2024-11-15 04:39:34,751 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 04:39:34,766 main DEBUG Took 0.012443 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 04:39:34,766 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 04:39:34,767 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 04:39:34,768 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 04:39:34,769 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,782 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 04:39:34,803 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,806 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,808 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,810 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,811 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,812 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,813 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,814 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,815 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,816 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,817 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,818 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,818 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,819 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,819 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 04:39:34,822 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,822 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 04:39:34,824 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 04:39:34,826 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 04:39:34,829 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 04:39:34,829 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 04:39:34,831 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 04:39:34,832 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 04:39:34,843 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 04:39:34,846 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 04:39:34,849 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 04:39:34,849 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 04:39:34,850 main DEBUG createAppenders(={Console}) 2024-11-15 04:39:34,851 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-15 04:39:34,851 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 04:39:34,852 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-15 04:39:34,852 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 04:39:34,853 main DEBUG OutputStream closed 2024-11-15 04:39:34,853 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 04:39:34,853 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 04:39:34,854 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-15 04:39:34,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 04:39:34,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 04:39:34,980 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 04:39:34,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 04:39:34,982 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 04:39:34,983 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 04:39:34,983 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 04:39:34,984 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 04:39:34,984 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 04:39:34,984 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 04:39:34,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 04:39:34,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 04:39:34,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 04:39:34,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 04:39:34,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 04:39:34,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 04:39:34,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 04:39:34,989 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 04:39:34,992 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 04:39:34,993 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-15 04:39:34,993 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 04:39:34,994 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-15T04:39:35,379 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d 2024-11-15 04:39:35,382 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 04:39:35,382 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T04:39:35,394 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-15T04:39:35,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T04:39:35,452 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/cluster_a5d1cf9b-a665-2596-bed6-8ed0e5e2e7a7, deleteOnExit=true 2024-11-15T04:39:35,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T04:39:35,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/test.cache.data in system properties and HBase conf 2024-11-15T04:39:35,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T04:39:35,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/hadoop.log.dir in system properties and HBase conf 2024-11-15T04:39:35,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T04:39:35,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T04:39:35,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T04:39:35,583 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T04:39:35,720 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T04:39:35,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T04:39:35,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T04:39:35,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T04:39:35,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T04:39:35,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T04:39:35,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T04:39:35,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T04:39:35,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T04:39:35,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T04:39:35,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/nfs.dump.dir in system properties and HBase conf 2024-11-15T04:39:35,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/java.io.tmpdir in system properties and HBase conf 2024-11-15T04:39:35,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T04:39:35,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T04:39:35,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T04:39:37,015 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T04:39:37,114 INFO [Time-limited test {}] log.Log(170): Logging initialized @3286ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T04:39:37,218 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T04:39:37,314 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T04:39:37,349 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T04:39:37,350 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T04:39:37,351 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T04:39:37,364 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T04:39:37,367 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/hadoop.log.dir/,AVAILABLE} 2024-11-15T04:39:37,369 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T04:39:37,587 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/java.io.tmpdir/jetty-localhost-36659-hadoop-hdfs-3_4_1-tests_jar-_-any-13968007035377433347/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T04:39:37,597 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:36659} 2024-11-15T04:39:37,598 INFO [Time-limited test {}] server.Server(415): Started @3771ms 2024-11-15T04:39:38,204 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T04:39:38,213 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T04:39:38,214 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T04:39:38,215 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T04:39:38,215 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T04:39:38,216 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/hadoop.log.dir/,AVAILABLE} 2024-11-15T04:39:38,217 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T04:39:38,335 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/java.io.tmpdir/jetty-localhost-46263-hadoop-hdfs-3_4_1-tests_jar-_-any-3943438706167892387/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T04:39:38,336 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:46263} 2024-11-15T04:39:38,336 INFO [Time-limited test {}] server.Server(415): Started @4509ms 2024-11-15T04:39:38,386 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T04:39:39,198 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/cluster_a5d1cf9b-a665-2596-bed6-8ed0e5e2e7a7/data/data2/current/BP-622654536-172.17.0.2-1731645576553/current, will proceed with Du for space computation calculation, 2024-11-15T04:39:39,198 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/cluster_a5d1cf9b-a665-2596-bed6-8ed0e5e2e7a7/data/data1/current/BP-622654536-172.17.0.2-1731645576553/current, will proceed with Du for space computation calculation, 2024-11-15T04:39:39,231 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T04:39:39,275 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb8fa16d0ad47ecee with lease ID 0x6b3e57c62b5a8db6: Processing first storage report for DS-ab31a3c3-be9d-415b-8288-4101ca5cd146 from datanode DatanodeRegistration(127.0.0.1:35353, datanodeUuid=ba099ddd-b7d4-410e-9ed8-42045e285825, infoPort=38743, infoSecurePort=0, ipcPort=35331, storageInfo=lv=-57;cid=testClusterID;nsid=1376036773;c=1731645576553) 2024-11-15T04:39:39,277 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb8fa16d0ad47ecee with lease ID 0x6b3e57c62b5a8db6: from storage DS-ab31a3c3-be9d-415b-8288-4101ca5cd146 node DatanodeRegistration(127.0.0.1:35353, datanodeUuid=ba099ddd-b7d4-410e-9ed8-42045e285825, infoPort=38743, infoSecurePort=0, ipcPort=35331, storageInfo=lv=-57;cid=testClusterID;nsid=1376036773;c=1731645576553), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T04:39:39,278 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb8fa16d0ad47ecee with lease ID 0x6b3e57c62b5a8db6: Processing first storage report for DS-974d636e-0928-40d0-aa99-5a1e1bbaf265 from datanode DatanodeRegistration(127.0.0.1:35353, datanodeUuid=ba099ddd-b7d4-410e-9ed8-42045e285825, infoPort=38743, infoSecurePort=0, ipcPort=35331, storageInfo=lv=-57;cid=testClusterID;nsid=1376036773;c=1731645576553) 2024-11-15T04:39:39,278 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb8fa16d0ad47ecee with lease ID 0x6b3e57c62b5a8db6: from storage DS-974d636e-0928-40d0-aa99-5a1e1bbaf265 node DatanodeRegistration(127.0.0.1:35353, datanodeUuid=ba099ddd-b7d4-410e-9ed8-42045e285825, infoPort=38743, infoSecurePort=0, ipcPort=35331, storageInfo=lv=-57;cid=testClusterID;nsid=1376036773;c=1731645576553), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T04:39:39,364 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d 2024-11-15T04:39:39,427 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/cluster_a5d1cf9b-a665-2596-bed6-8ed0e5e2e7a7/zookeeper_0, clientPort=51527, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/cluster_a5d1cf9b-a665-2596-bed6-8ed0e5e2e7a7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/cluster_a5d1cf9b-a665-2596-bed6-8ed0e5e2e7a7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T04:39:39,435 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51527 2024-11-15T04:39:39,449 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T04:39:39,453 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T04:39:39,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741825_1001 (size=7) 2024-11-15T04:39:40,065 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6 with version=8 2024-11-15T04:39:40,065 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/hbase-staging 2024-11-15T04:39:40,144 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T04:39:40,421 INFO [Time-limited test {}] client.ConnectionUtils(128): master/26747e6ada4f:0 server-side Connection retries=45 2024-11-15T04:39:40,430 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T04:39:40,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T04:39:40,436 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T04:39:40,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T04:39:40,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T04:39:40,583 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T04:39:40,641 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T04:39:40,651 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T04:39:40,656 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T04:39:40,688 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 996 (auto-detected) 2024-11-15T04:39:40,689 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-15T04:39:40,708 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42655 2024-11-15T04:39:40,737 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42655 connecting to ZooKeeper ensemble=127.0.0.1:51527 2024-11-15T04:39:40,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:426550x0, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T04:39:40,848 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42655-0x1013e1433580000 connected 2024-11-15T04:39:40,934 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T04:39:40,937 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T04:39:40,947 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T04:39:40,952 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6, hbase.cluster.distributed=false 2024-11-15T04:39:40,980 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T04:39:40,994 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42655 2024-11-15T04:39:40,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42655 2024-11-15T04:39:40,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42655 2024-11-15T04:39:40,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42655 2024-11-15T04:39:40,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42655 2024-11-15T04:39:41,118 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/26747e6ada4f:0 server-side Connection retries=45 2024-11-15T04:39:41,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T04:39:41,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T04:39:41,120 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T04:39:41,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T04:39:41,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T04:39:41,123 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T04:39:41,125 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T04:39:41,126 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42935 2024-11-15T04:39:41,128 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42935 connecting to ZooKeeper ensemble=127.0.0.1:51527 2024-11-15T04:39:41,130 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T04:39:41,133 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T04:39:41,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429350x0, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T04:39:41,166 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:429350x0, quorum=127.0.0.1:51527, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T04:39:41,166 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42935-0x1013e1433580001 connected 2024-11-15T04:39:41,169 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T04:39:41,176 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T04:39:41,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T04:39:41,183 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T04:39:41,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42935 2024-11-15T04:39:41,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42935 2024-11-15T04:39:41,184 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42935 2024-11-15T04:39:41,185 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42935 2024-11-15T04:39:41,185 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42935 2024-11-15T04:39:41,200 DEBUG [M:0;26747e6ada4f:42655 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;26747e6ada4f:42655 2024-11-15T04:39:41,201 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/26747e6ada4f,42655,1731645580243 2024-11-15T04:39:41,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T04:39:41,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T04:39:41,233 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/26747e6ada4f,42655,1731645580243 2024-11-15T04:39:41,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:41,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T04:39:41,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:41,266 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T04:39:41,267 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/26747e6ada4f,42655,1731645580243 from backup master directory 2024-11-15T04:39:41,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T04:39:41,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/26747e6ada4f,42655,1731645580243 2024-11-15T04:39:41,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T04:39:41,280 WARN [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T04:39:41,280 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=26747e6ada4f,42655,1731645580243 2024-11-15T04:39:41,283 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T04:39:41,284 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T04:39:41,340 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/hbase.id] with ID: fb97bd75-b06f-46e2-9d7c-0f171115d342 2024-11-15T04:39:41,340 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/.tmp/hbase.id 2024-11-15T04:39:41,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741826_1002 (size=42) 2024-11-15T04:39:41,756 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/.tmp/hbase.id]:[hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/hbase.id] 2024-11-15T04:39:41,803 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T04:39:41,810 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T04:39:41,829 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-15T04:39:41,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:41,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:41,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741827_1003 (size=196) 2024-11-15T04:39:42,321 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T04:39:42,324 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T04:39:42,337 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:42,341 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T04:39:42,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741828_1004 (size=1189) 2024-11-15T04:39:42,783 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store 2024-11-15T04:39:42,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741829_1005 (size=34) 2024-11-15T04:39:43,206 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T04:39:43,209 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:39:43,210 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T04:39:43,211 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T04:39:43,211 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T04:39:43,212 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T04:39:43,213 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T04:39:43,213 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T04:39:43,214 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731645583210Disabling compacts and flushes for region at 1731645583210Disabling writes for close at 1731645583213 (+3 ms)Writing region close event to WAL at 1731645583213Closed at 1731645583213 2024-11-15T04:39:43,217 WARN [master/26747e6ada4f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/.initializing 2024-11-15T04:39:43,217 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/WALs/26747e6ada4f,42655,1731645580243 2024-11-15T04:39:43,226 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T04:39:43,242 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26747e6ada4f%2C42655%2C1731645580243, suffix=, logDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/WALs/26747e6ada4f,42655,1731645580243, archiveDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/oldWALs, maxLogs=10 2024-11-15T04:39:43,265 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/WALs/26747e6ada4f,42655,1731645580243/26747e6ada4f%2C42655%2C1731645580243.1731645583247, exclude list is [], retry=0 2024-11-15T04:39:43,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35353,DS-ab31a3c3-be9d-415b-8288-4101ca5cd146,DISK] 2024-11-15T04:39:43,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T04:39:43,323 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/WALs/26747e6ada4f,42655,1731645580243/26747e6ada4f%2C42655%2C1731645580243.1731645583247 2024-11-15T04:39:43,324 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38743:38743)] 2024-11-15T04:39:43,325 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:39:43,325 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:39:43,329 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,330 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,368 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,397 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T04:39:43,400 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:43,403 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T04:39:43,404 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T04:39:43,408 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:43,409 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:39:43,409 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,413 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T04:39:43,413 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:43,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:39:43,415 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T04:39:43,419 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:43,421 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:39:43,421 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,430 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,432 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,439 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,440 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,444 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T04:39:43,449 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T04:39:43,454 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T04:39:43,456 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73147679, jitterRate=0.08998535573482513}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T04:39:43,463 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731645583343Initializing all the Stores at 1731645583345 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731645583346 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645583347 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645583347Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645583347Cleaning up temporary data from old regions at 1731645583440 (+93 ms)Region opened successfully at 1731645583463 (+23 ms) 2024-11-15T04:39:43,466 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T04:39:43,502 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77884a15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26747e6ada4f/172.17.0.2:0 2024-11-15T04:39:43,536 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T04:39:43,552 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T04:39:43,552 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T04:39:43,556 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T04:39:43,558 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T04:39:43,565 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T04:39:43,565 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T04:39:43,595 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T04:39:43,604 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T04:39:43,656 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T04:39:43,659 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T04:39:43,661 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T04:39:43,672 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T04:39:43,675 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T04:39:43,680 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T04:39:43,689 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T04:39:43,691 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T04:39:43,697 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T04:39:43,713 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T04:39:43,722 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T04:39:43,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T04:39:43,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T04:39:43,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:43,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:43,733 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=26747e6ada4f,42655,1731645580243, sessionid=0x1013e1433580000, setting cluster-up flag (Was=false) 2024-11-15T04:39:43,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:43,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:43,781 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T04:39:43,783 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=26747e6ada4f,42655,1731645580243 2024-11-15T04:39:43,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:43,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:43,831 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T04:39:43,835 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=26747e6ada4f,42655,1731645580243 2024-11-15T04:39:43,843 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T04:39:43,892 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(746): ClusterId : fb97bd75-b06f-46e2-9d7c-0f171115d342 2024-11-15T04:39:43,894 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T04:39:43,908 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T04:39:43,916 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T04:39:43,917 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T04:39:43,918 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T04:39:43,924 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T04:39:43,924 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T04:39:43,925 DEBUG [RS:0;26747e6ada4f:42935 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa46060, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26747e6ada4f/172.17.0.2:0 2024-11-15T04:39:43,929 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 26747e6ada4f,42655,1731645580243 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T04:39:43,936 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/26747e6ada4f:0, corePoolSize=5, maxPoolSize=5 2024-11-15T04:39:43,936 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/26747e6ada4f:0, corePoolSize=5, maxPoolSize=5 2024-11-15T04:39:43,936 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/26747e6ada4f:0, corePoolSize=5, maxPoolSize=5 2024-11-15T04:39:43,936 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/26747e6ada4f:0, corePoolSize=5, maxPoolSize=5 2024-11-15T04:39:43,936 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/26747e6ada4f:0, corePoolSize=10, maxPoolSize=10 2024-11-15T04:39:43,936 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:43,937 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/26747e6ada4f:0, corePoolSize=2, maxPoolSize=2 2024-11-15T04:39:43,937 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:43,938 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731645613938 2024-11-15T04:39:43,939 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;26747e6ada4f:42935 2024-11-15T04:39:43,939 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T04:39:43,940 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T04:39:43,942 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T04:39:43,942 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T04:39:43,942 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T04:39:43,942 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T04:39:43,943 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T04:39:43,944 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T04:39:43,944 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T04:39:43,944 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T04:39:43,945 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(2659): reportForDuty to master=26747e6ada4f,42655,1731645580243 with port=42935, startcode=1731645581086 2024-11-15T04:39:43,945 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T04:39:43,945 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:43,948 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T04:39:43,948 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:43,949 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T04:39:43,949 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T04:39:43,950 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T04:39:43,951 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T04:39:43,952 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T04:39:43,954 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/26747e6ada4f:0:becomeActiveMaster-HFileCleaner.large.0-1731645583953,5,FailOnTimeoutGroup] 2024-11-15T04:39:43,955 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/26747e6ada4f:0:becomeActiveMaster-HFileCleaner.small.0-1731645583954,5,FailOnTimeoutGroup] 2024-11-15T04:39:43,955 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:43,955 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T04:39:43,956 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:43,957 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:43,957 DEBUG [RS:0;26747e6ada4f:42935 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T04:39:43,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741831_1007 (size=1321) 2024-11-15T04:39:43,990 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40609, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T04:39:43,995 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42655 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 26747e6ada4f,42935,1731645581086 2024-11-15T04:39:43,997 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42655 {}] master.ServerManager(517): Registering regionserver=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:44,010 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6 2024-11-15T04:39:44,010 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38061 2024-11-15T04:39:44,011 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T04:39:44,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T04:39:44,046 DEBUG [RS:0;26747e6ada4f:42935 {}] zookeeper.ZKUtil(111): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/26747e6ada4f,42935,1731645581086 2024-11-15T04:39:44,047 WARN [RS:0;26747e6ada4f:42935 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T04:39:44,047 INFO [RS:0;26747e6ada4f:42935 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T04:39:44,047 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/WALs/26747e6ada4f,42935,1731645581086 2024-11-15T04:39:44,049 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [26747e6ada4f,42935,1731645581086] 2024-11-15T04:39:44,076 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T04:39:44,091 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T04:39:44,096 INFO [RS:0;26747e6ada4f:42935 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T04:39:44,096 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:44,097 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T04:39:44,102 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T04:39:44,103 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:44,104 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,104 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,104 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,104 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,105 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,105 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/26747e6ada4f:0, corePoolSize=2, maxPoolSize=2 2024-11-15T04:39:44,105 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,105 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,105 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,106 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,106 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,106 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/26747e6ada4f:0, corePoolSize=1, maxPoolSize=1 2024-11-15T04:39:44,106 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/26747e6ada4f:0, corePoolSize=3, maxPoolSize=3 2024-11-15T04:39:44,107 DEBUG [RS:0;26747e6ada4f:42935 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0, corePoolSize=3, maxPoolSize=3 2024-11-15T04:39:44,108 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:44,108 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:44,108 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:44,108 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:44,108 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:44,108 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=26747e6ada4f,42935,1731645581086-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T04:39:44,130 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T04:39:44,132 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=26747e6ada4f,42935,1731645581086-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:44,132 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:44,132 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.Replication(171): 26747e6ada4f,42935,1731645581086 started 2024-11-15T04:39:44,152 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:44,153 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1482): Serving as 26747e6ada4f,42935,1731645581086, RpcServer on 26747e6ada4f/172.17.0.2:42935, sessionid=0x1013e1433580001 2024-11-15T04:39:44,154 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T04:39:44,154 DEBUG [RS:0;26747e6ada4f:42935 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 26747e6ada4f,42935,1731645581086 2024-11-15T04:39:44,154 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26747e6ada4f,42935,1731645581086' 2024-11-15T04:39:44,154 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T04:39:44,155 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T04:39:44,156 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T04:39:44,156 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T04:39:44,156 DEBUG [RS:0;26747e6ada4f:42935 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 26747e6ada4f,42935,1731645581086 2024-11-15T04:39:44,156 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26747e6ada4f,42935,1731645581086' 2024-11-15T04:39:44,156 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T04:39:44,157 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T04:39:44,158 DEBUG [RS:0;26747e6ada4f:42935 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T04:39:44,158 INFO [RS:0;26747e6ada4f:42935 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T04:39:44,158 INFO [RS:0;26747e6ada4f:42935 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T04:39:44,264 INFO [RS:0;26747e6ada4f:42935 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T04:39:44,269 INFO [RS:0;26747e6ada4f:42935 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26747e6ada4f%2C42935%2C1731645581086, suffix=, logDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/WALs/26747e6ada4f,42935,1731645581086, archiveDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/oldWALs, maxLogs=32 2024-11-15T04:39:44,284 DEBUG [RS:0;26747e6ada4f:42935 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/WALs/26747e6ada4f,42935,1731645581086/26747e6ada4f%2C42935%2C1731645581086.1731645584272, exclude list is [], retry=0 2024-11-15T04:39:44,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35353,DS-ab31a3c3-be9d-415b-8288-4101ca5cd146,DISK] 2024-11-15T04:39:44,292 INFO [RS:0;26747e6ada4f:42935 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/WALs/26747e6ada4f,42935,1731645581086/26747e6ada4f%2C42935%2C1731645581086.1731645584272 2024-11-15T04:39:44,293 DEBUG [RS:0;26747e6ada4f:42935 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38743:38743)] 2024-11-15T04:39:44,362 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T04:39:44,363 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6 2024-11-15T04:39:44,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741833_1009 (size=32) 2024-11-15T04:39:44,784 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:39:44,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T04:39:44,793 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T04:39:44,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:44,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T04:39:44,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T04:39:44,797 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T04:39:44,797 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:44,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T04:39:44,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T04:39:44,801 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T04:39:44,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:44,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T04:39:44,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T04:39:44,804 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T04:39:44,804 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:44,805 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T04:39:44,805 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T04:39:44,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740 2024-11-15T04:39:44,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740 2024-11-15T04:39:44,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T04:39:44,810 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T04:39:44,811 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:39:44,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T04:39:44,816 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T04:39:44,817 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75346578, jitterRate=0.12275150418281555}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:39:44,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731645584784Initializing all the Stores at 1731645584789 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731645584789Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731645584789Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645584790 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731645584790Cleaning up temporary data from old regions at 1731645584810 (+20 ms)Region opened successfully at 1731645584820 (+10 ms) 2024-11-15T04:39:44,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T04:39:44,820 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T04:39:44,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T04:39:44,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T04:39:44,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T04:39:44,821 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T04:39:44,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731645584820Disabling compacts and flushes for region at 1731645584820Disabling writes for close at 1731645584820Writing region close event to WAL at 1731645584821 (+1 ms)Closed at 1731645584821 2024-11-15T04:39:44,825 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T04:39:44,825 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T04:39:44,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T04:39:44,838 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T04:39:44,840 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T04:39:44,997 DEBUG [26747e6ada4f:42655 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T04:39:45,011 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:45,017 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26747e6ada4f,42935,1731645581086, state=OPENING 2024-11-15T04:39:45,073 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T04:39:45,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:45,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:39:45,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T04:39:45,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T04:39:45,084 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T04:39:45,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:39:45,266 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T04:39:45,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51009, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T04:39:45,279 INFO [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T04:39:45,279 INFO [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T04:39:45,280 INFO [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T04:39:45,283 INFO [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26747e6ada4f%2C42935%2C1731645581086.meta, suffix=.meta, logDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/WALs/26747e6ada4f,42935,1731645581086, archiveDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/oldWALs, maxLogs=32 2024-11-15T04:39:45,299 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/WALs/26747e6ada4f,42935,1731645581086/26747e6ada4f%2C42935%2C1731645581086.meta.1731645585286.meta, exclude list is [], retry=0 2024-11-15T04:39:45,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35353,DS-ab31a3c3-be9d-415b-8288-4101ca5cd146,DISK] 2024-11-15T04:39:45,305 INFO [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/WALs/26747e6ada4f,42935,1731645581086/26747e6ada4f%2C42935%2C1731645581086.meta.1731645585286.meta 2024-11-15T04:39:45,306 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38743:38743)] 2024-11-15T04:39:45,306 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:39:45,307 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T04:39:45,310 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T04:39:45,314 INFO [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T04:39:45,318 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T04:39:45,318 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:39:45,319 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T04:39:45,319 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T04:39:45,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T04:39:45,324 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T04:39:45,324 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:45,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T04:39:45,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T04:39:45,327 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T04:39:45,327 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:45,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T04:39:45,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T04:39:45,330 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T04:39:45,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:45,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T04:39:45,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T04:39:45,332 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T04:39:45,332 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:45,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T04:39:45,334 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T04:39:45,335 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740 2024-11-15T04:39:45,337 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740 2024-11-15T04:39:45,340 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T04:39:45,340 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T04:39:45,341 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:39:45,343 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T04:39:45,345 INFO [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61406527, jitterRate=-0.08497144281864166}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:39:45,345 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T04:39:45,347 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731645585319Writing region info on filesystem at 1731645585319Initializing all the Stores at 1731645585321 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731645585321Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731645585322 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645585322Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731645585322Cleaning up temporary data from old regions at 1731645585340 (+18 ms)Running coprocessor post-open hooks at 1731645585345 (+5 ms)Region opened successfully at 1731645585347 (+2 ms) 2024-11-15T04:39:45,353 INFO [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731645585257 2024-11-15T04:39:45,364 DEBUG [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T04:39:45,365 INFO [RS_OPEN_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T04:39:45,367 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:45,370 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26747e6ada4f,42935,1731645581086, state=OPEN 2024-11-15T04:39:45,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T04:39:45,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T04:39:45,431 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T04:39:45,431 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T04:39:45,431 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:45,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T04:39:45,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=26747e6ada4f,42935,1731645581086 in 347 msec 2024-11-15T04:39:45,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T04:39:45,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 609 msec 2024-11-15T04:39:45,448 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T04:39:45,448 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T04:39:45,470 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:45,471 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:45,489 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:45,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44481, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:45,513 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6400 sec 2024-11-15T04:39:45,513 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731645585513, completionTime=-1 2024-11-15T04:39:45,515 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T04:39:45,515 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T04:39:45,539 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T04:39:45,539 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731645645539 2024-11-15T04:39:45,539 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731645705539 2024-11-15T04:39:45,539 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-15T04:39:45,541 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26747e6ada4f,42655,1731645580243-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:45,541 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26747e6ada4f,42655,1731645580243-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:45,541 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26747e6ada4f,42655,1731645580243-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:45,542 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-26747e6ada4f:42655, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:45,542 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:45,543 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:45,548 DEBUG [master/26747e6ada4f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T04:39:45,568 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.287sec 2024-11-15T04:39:45,569 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T04:39:45,570 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T04:39:45,571 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T04:39:45,571 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T04:39:45,572 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T04:39:45,572 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26747e6ada4f,42655,1731645580243-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T04:39:45,573 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26747e6ada4f,42655,1731645580243-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T04:39:45,581 DEBUG [master/26747e6ada4f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T04:39:45,582 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T04:39:45,582 INFO [master/26747e6ada4f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26747e6ada4f,42655,1731645580243-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T04:39:45,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bed8dc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:45,605 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T04:39:45,605 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T04:39:45,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:45,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:45,617 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:45,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:45,626 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:45,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738d23fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:45,629 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:45,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:45,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=26747e6ada4f,42655,1731645580243 2024-11-15T04:39:45,697 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=238, ProcessCount=12, AvailableMemoryMB=15407 2024-11-15T04:39:45,707 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T04:39:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 26747e6ada4f,42655,1731645580243 2024-11-15T04:39:45,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56ef0928 2024-11-15T04:39:45,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T04:39:45,721 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T04:39:45,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T04:39:45,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T04:39:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T04:39:45,738 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T04:39:45,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T04:39:45,741 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:45,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T04:39:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T04:39:45,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741835_1011 (size=959) 2024-11-15T04:39:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T04:39:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T04:39:46,163 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6 2024-11-15T04:39:46,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741836_1012 (size=53) 2024-11-15T04:39:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T04:39:46,579 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:39:46,580 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bffc5fba59ef21ffa1864f468203f7b0, disabling compactions & flushes 2024-11-15T04:39:46,580 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:46,581 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:46,581 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. after waiting 0 ms 2024-11-15T04:39:46,581 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:46,581 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:46,582 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bffc5fba59ef21ffa1864f468203f7b0: Waiting for close lock at 1731645586580Disabling compacts and flushes for region at 1731645586580Disabling writes for close at 1731645586581 (+1 ms)Writing region close event to WAL at 1731645586581Closed at 1731645586581 2024-11-15T04:39:46,587 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T04:39:46,592 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731645586587"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731645586587"}]},"ts":"1731645586587"} 2024-11-15T04:39:46,596 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T04:39:46,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T04:39:46,601 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645586598"}]},"ts":"1731645586598"} 2024-11-15T04:39:46,605 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T04:39:46,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bffc5fba59ef21ffa1864f468203f7b0, ASSIGN}] 2024-11-15T04:39:46,610 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bffc5fba59ef21ffa1864f468203f7b0, ASSIGN 2024-11-15T04:39:46,612 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bffc5fba59ef21ffa1864f468203f7b0, ASSIGN; state=OFFLINE, location=26747e6ada4f,42935,1731645581086; forceNewPlan=false, retain=false 2024-11-15T04:39:46,764 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bffc5fba59ef21ffa1864f468203f7b0, regionState=OPENING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:46,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bffc5fba59ef21ffa1864f468203f7b0, ASSIGN because future has completed 2024-11-15T04:39:46,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:39:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T04:39:46,937 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:46,937 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:39:46,937 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,937 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:39:46,938 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,938 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,940 INFO [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,942 INFO [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bffc5fba59ef21ffa1864f468203f7b0 columnFamilyName A 2024-11-15T04:39:46,942 DEBUG [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:46,944 INFO [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] regionserver.HStore(327): Store=bffc5fba59ef21ffa1864f468203f7b0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:39:46,944 INFO [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,946 INFO [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bffc5fba59ef21ffa1864f468203f7b0 columnFamilyName B 2024-11-15T04:39:46,946 DEBUG [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:46,947 INFO [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] regionserver.HStore(327): Store=bffc5fba59ef21ffa1864f468203f7b0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:39:46,947 INFO [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,949 INFO [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bffc5fba59ef21ffa1864f468203f7b0 columnFamilyName C 2024-11-15T04:39:46,949 DEBUG [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:39:46,950 INFO [StoreOpener-bffc5fba59ef21ffa1864f468203f7b0-1 {}] regionserver.HStore(327): Store=bffc5fba59ef21ffa1864f468203f7b0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:39:46,950 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,952 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,953 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,955 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,956 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,957 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:39:46,959 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,963 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T04:39:46,964 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bffc5fba59ef21ffa1864f468203f7b0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72299172, jitterRate=0.07734161615371704}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:39:46,964 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:46,965 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bffc5fba59ef21ffa1864f468203f7b0: Running coprocessor pre-open hook at 1731645586938Writing region info on filesystem at 1731645586938Initializing all the Stores at 1731645586940 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645586940Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645586940Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645586940Cleaning up temporary data from old regions at 1731645586956 (+16 ms)Running coprocessor post-open hooks at 1731645586964 (+8 ms)Region opened successfully at 1731645586965 (+1 ms) 2024-11-15T04:39:46,967 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., pid=6, masterSystemTime=1731645586928 2024-11-15T04:39:46,970 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:46,971 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:46,972 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bffc5fba59ef21ffa1864f468203f7b0, regionState=OPEN, openSeqNum=2, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:46,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:39:46,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T04:39:46,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 in 207 msec 2024-11-15T04:39:46,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T04:39:46,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bffc5fba59ef21ffa1864f468203f7b0, ASSIGN in 376 msec 2024-11-15T04:39:46,989 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T04:39:46,989 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645586989"}]},"ts":"1731645586989"} 2024-11-15T04:39:46,992 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T04:39:46,994 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T04:39:46,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2650 sec 2024-11-15T04:39:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T04:39:47,895 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:39:47,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ffdeb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:47,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:47,906 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:47,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:47,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:47,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ca7478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:47,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:47,909 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:47,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38922b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,911 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:47,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:47,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:47,916 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:47,918 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T04:39:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 26747e6ada4f,42655,1731645580243 2024-11-15T04:39:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28ec11fd 2024-11-15T04:39:47,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T04:39:47,925 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T04:39:47,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ce0b30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:47,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:47,931 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71374046, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:47,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:47,935 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:47,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43659da1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15170e84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:47,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:47,938 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180c72f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:47,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:47,942 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:47,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745237a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b6c8cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:47,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:47,946 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@174557e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:47,949 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71fbbc14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:47,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:47,953 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:47,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:47,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:47,956 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77447b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35763efb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:47,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:47,959 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:47,962 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:47,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25318c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3162141e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:47,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:47,965 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:47,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:47,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:47,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:47,968 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:47,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e1b379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27f30934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:47,970 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:47,971 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:47,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:47,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f9dc53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:47,973 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:47,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c1f4fd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7371bc35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:47,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:47,977 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd08b46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:47,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:47,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:47,979 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:47,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd5f624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9cd7d7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:39:47,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:39:47,983 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:39:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:39:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:39:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2269f6e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,983 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:39:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:39:47,984 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:39:47,985 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:39:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16bc4caa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:39:47,989 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:47,989 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:47,989 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:47,990 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:47,991 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:47,992 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:47,992 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:47,992 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:39:47,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:47,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:47,994 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:47,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:47,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:47,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:47,996 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:39:47,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:47,997 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:47,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:48,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:48,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:39:48,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:39:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:39:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:39:48,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:39:48,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:48,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:48,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:39:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:39:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:39:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T04:39:48,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:39:48,012 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:39:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T04:39:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:39:48,016 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:39:48,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:39:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:39:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:39:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:48,104 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:39:48,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T04:39:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T04:39:48,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:48,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:48,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:48,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:48,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/68b108ea5d544798b0f0e3388ec42b0c is 50, key is test_row_0/A:col10/1731645588106/Put/seqid=0 2024-11-15T04:39:48,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741837_1013 (size=19021) 2024-11-15T04:39:48,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645648258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645648267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645648299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645648307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645648312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T04:39:48,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:48,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:48,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:48,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:48,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645648464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645648465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645648465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645648465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645648466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:48,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T04:39:48,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645648673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645648684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645648685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645648686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645648686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/68b108ea5d544798b0f0e3388ec42b0c 2024-11-15T04:39:48,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:48,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:48,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:48,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:48,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/81b305d66b26498696f95f50efeab58f is 50, key is test_row_0/B:col10/1731645588106/Put/seqid=0 2024-11-15T04:39:48,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741838_1014 (size=12001) 2024-11-15T04:39:48,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/81b305d66b26498696f95f50efeab58f 2024-11-15T04:39:48,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/16836675119a4dd38fd7d36b188993d0 is 50, key is test_row_0/C:col10/1731645588106/Put/seqid=0 2024-11-15T04:39:48,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:48,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:48,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:48,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:48,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:48,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741839_1015 (size=12001) 2024-11-15T04:39:48,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645648983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645648994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645648994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645648995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:48,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645648995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:49,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:49,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T04:39:49,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:49,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/16836675119a4dd38fd7d36b188993d0 2024-11-15T04:39:49,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/68b108ea5d544798b0f0e3388ec42b0c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/68b108ea5d544798b0f0e3388ec42b0c 2024-11-15T04:39:49,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/68b108ea5d544798b0f0e3388ec42b0c, entries=300, sequenceid=14, filesize=18.6 K 2024-11-15T04:39:49,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/81b305d66b26498696f95f50efeab58f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/81b305d66b26498696f95f50efeab58f 2024-11-15T04:39:49,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:49,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:49,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/81b305d66b26498696f95f50efeab58f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T04:39:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/16836675119a4dd38fd7d36b188993d0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/16836675119a4dd38fd7d36b188993d0 2024-11-15T04:39:49,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/16836675119a4dd38fd7d36b188993d0, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T04:39:49,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for bffc5fba59ef21ffa1864f468203f7b0 in 1279ms, sequenceid=14, compaction requested=false 2024-11-15T04:39:49,392 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T04:39:49,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:49,501 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:39:49,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:39:49,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:49,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:49,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/fde0a0dbf00e407594e6d7199e9d2795 is 50, key is test_row_0/A:col10/1731645588297/Put/seqid=0 2024-11-15T04:39:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741840_1016 (size=14341) 2024-11-15T04:39:49,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/fde0a0dbf00e407594e6d7199e9d2795 2024-11-15T04:39:49,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/968235c733c04d809a374bf985d5cb5f is 50, key is test_row_0/B:col10/1731645588297/Put/seqid=0 2024-11-15T04:39:49,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741841_1017 (size=12001) 2024-11-15T04:39:49,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/968235c733c04d809a374bf985d5cb5f 2024-11-15T04:39:49,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/8ae82e46226940d4b6df324400f53cc5 is 50, key is test_row_0/C:col10/1731645588297/Put/seqid=0 2024-11-15T04:39:49,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741842_1018 (size=12001) 2024-11-15T04:39:49,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645649603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645649603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645649614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645649610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645649618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:49,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645649736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645649737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645649737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645649737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645649743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:49,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645649953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645649963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645649975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645649975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645649975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:49,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:49,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:49,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:49,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:49,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:49,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/8ae82e46226940d4b6df324400f53cc5 2024-11-15T04:39:50,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/fde0a0dbf00e407594e6d7199e9d2795 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fde0a0dbf00e407594e6d7199e9d2795 2024-11-15T04:39:50,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fde0a0dbf00e407594e6d7199e9d2795, entries=200, sequenceid=39, filesize=14.0 K 2024-11-15T04:39:50,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/968235c733c04d809a374bf985d5cb5f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/968235c733c04d809a374bf985d5cb5f 2024-11-15T04:39:50,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/968235c733c04d809a374bf985d5cb5f, entries=150, sequenceid=39, filesize=11.7 K 2024-11-15T04:39:50,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/8ae82e46226940d4b6df324400f53cc5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8ae82e46226940d4b6df324400f53cc5 2024-11-15T04:39:50,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8ae82e46226940d4b6df324400f53cc5, entries=150, sequenceid=39, filesize=11.7 K 2024-11-15T04:39:50,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for bffc5fba59ef21ffa1864f468203f7b0 in 562ms, sequenceid=39, compaction requested=false 2024-11-15T04:39:50,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:50,070 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T04:39:50,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T04:39:50,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:50,144 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:39:50,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/9ea27444b32b4a84a13004a8cfd7e795 is 50, key is test_row_0/A:col10/1731645589600/Put/seqid=0 2024-11-15T04:39:50,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T04:39:50,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741843_1019 (size=9657) 2024-11-15T04:39:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:50,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645650359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645650362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645650367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645650373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645650384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645650483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645650494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645650494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645650494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645650503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,580 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/9ea27444b32b4a84a13004a8cfd7e795 2024-11-15T04:39:50,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/44aa9f1365964518a0325db644b30d4e is 50, key is test_row_0/B:col10/1731645589600/Put/seqid=0 2024-11-15T04:39:50,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741844_1020 (size=9657) 2024-11-15T04:39:50,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645650703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645650713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645650713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645650713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:50,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645650715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:50,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:50,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:50,837 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T04:39:50,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T04:39:50,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T04:39:50,958 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T04:39:50,958 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T04:39:50,973 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T04:39:50,973 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T04:39:50,974 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T04:39:50,974 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T04:39:51,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645651023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:51,031 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/44aa9f1365964518a0325db644b30d4e 2024-11-15T04:39:51,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645651034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:51,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645651034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645651034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:51,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645651034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:51,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/99e9d5f8403b4ef495fc2b09890e638e is 50, key is test_row_0/C:col10/1731645589600/Put/seqid=0 2024-11-15T04:39:51,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741845_1021 (size=9657) 2024-11-15T04:39:51,509 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/99e9d5f8403b4ef495fc2b09890e638e 2024-11-15T04:39:51,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/9ea27444b32b4a84a13004a8cfd7e795 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9ea27444b32b4a84a13004a8cfd7e795 2024-11-15T04:39:51,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645651533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:51,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645651543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:51,546 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9ea27444b32b4a84a13004a8cfd7e795, entries=100, sequenceid=50, filesize=9.4 K 2024-11-15T04:39:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:51,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/44aa9f1365964518a0325db644b30d4e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/44aa9f1365964518a0325db644b30d4e 2024-11-15T04:39:51,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645651554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:51,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645651554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:51,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645651554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:51,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:51,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:51,566 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/44aa9f1365964518a0325db644b30d4e, entries=100, sequenceid=50, filesize=9.4 K 2024-11-15T04:39:51,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/99e9d5f8403b4ef495fc2b09890e638e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/99e9d5f8403b4ef495fc2b09890e638e 2024-11-15T04:39:51,587 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/99e9d5f8403b4ef495fc2b09890e638e, entries=100, sequenceid=50, filesize=9.4 K 2024-11-15T04:39:51,589 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for bffc5fba59ef21ffa1864f468203f7b0 in 1445ms, sequenceid=50, compaction requested=true 2024-11-15T04:39:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:51,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:51,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T04:39:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T04:39:51,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T04:39:51,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5720 sec 2024-11-15T04:39:51,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.6040 sec 2024-11-15T04:39:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T04:39:52,184 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:39:52,184 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:39:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T04:39:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T04:39:52,197 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:39:52,199 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:39:52,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:39:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T04:39:52,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T04:39:52,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:52,361 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T04:39:52,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/ed4d264c193d4aa181b79109192ba905 is 50, key is test_row_0/A:col10/1731645590356/Put/seqid=0 2024-11-15T04:39:52,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741846_1022 (size=12001) 2024-11-15T04:39:52,448 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/ed4d264c193d4aa181b79109192ba905 2024-11-15T04:39:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/50e1c5e4a6194459ace2e56258bb6f23 is 50, key is test_row_0/B:col10/1731645590356/Put/seqid=0 2024-11-15T04:39:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T04:39:52,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741847_1023 (size=12001) 2024-11-15T04:39:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:52,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:52,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645652590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645652591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645652592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645652594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645652601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645652713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645652714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645652714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645652714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645652717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T04:39:52,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645652925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645652925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645652933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645652933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645652935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:52,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:52,953 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/50e1c5e4a6194459ace2e56258bb6f23 2024-11-15T04:39:53,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/24232a027c744cd0b7ba5157260cdfc7 is 50, key is test_row_0/C:col10/1731645590356/Put/seqid=0 2024-11-15T04:39:53,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741848_1024 (size=12001) 2024-11-15T04:39:53,061 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/24232a027c744cd0b7ba5157260cdfc7 2024-11-15T04:39:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/ed4d264c193d4aa181b79109192ba905 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/ed4d264c193d4aa181b79109192ba905 2024-11-15T04:39:53,096 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/ed4d264c193d4aa181b79109192ba905, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T04:39:53,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/50e1c5e4a6194459ace2e56258bb6f23 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/50e1c5e4a6194459ace2e56258bb6f23 2024-11-15T04:39:53,130 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/50e1c5e4a6194459ace2e56258bb6f23, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T04:39:53,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/24232a027c744cd0b7ba5157260cdfc7 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/24232a027c744cd0b7ba5157260cdfc7 2024-11-15T04:39:53,158 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/24232a027c744cd0b7ba5157260cdfc7, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T04:39:53,163 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for bffc5fba59ef21ffa1864f468203f7b0 in 803ms, sequenceid=76, compaction requested=true 2024-11-15T04:39:53,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:53,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:53,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T04:39:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T04:39:53,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T04:39:53,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 967 msec 2024-11-15T04:39:53,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 991 msec 2024-11-15T04:39:53,249 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:39:53,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T04:39:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:53,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/50bd26e121f848f4a42b9aed77cf4814 is 50, key is test_row_0/A:col10/1731645593248/Put/seqid=0 2024-11-15T04:39:53,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741849_1025 (size=16681) 2024-11-15T04:39:53,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/50bd26e121f848f4a42b9aed77cf4814 2024-11-15T04:39:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T04:39:53,344 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:39:53,345 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:39:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T04:39:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T04:39:53,349 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:39:53,353 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:39:53,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:39:53,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/32cfffec15484fa3a8f56d6952c4c28e is 50, key is test_row_0/B:col10/1731645593248/Put/seqid=0 2024-11-15T04:39:53,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645653377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645653378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645653382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645653383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645653383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741850_1026 (size=12001) 2024-11-15T04:39:53,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/32cfffec15484fa3a8f56d6952c4c28e 2024-11-15T04:39:53,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/4052a0e9f19845fe96ce7e6a86ac9af7 is 50, key is test_row_0/C:col10/1731645593248/Put/seqid=0 2024-11-15T04:39:53,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T04:39:53,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741851_1027 (size=12001) 2024-11-15T04:39:53,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645653493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645653494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645653493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645653494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T04:39:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:53,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:53,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645653494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T04:39:53,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T04:39:53,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:53,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:53,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:53,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:53,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:53,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645653715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645653715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645653715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645653715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645653733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:53,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T04:39:53,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:53,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:53,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:53,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:53,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:53,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/4052a0e9f19845fe96ce7e6a86ac9af7 2024-11-15T04:39:53,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/50bd26e121f848f4a42b9aed77cf4814 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/50bd26e121f848f4a42b9aed77cf4814 2024-11-15T04:39:53,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/50bd26e121f848f4a42b9aed77cf4814, entries=250, sequenceid=89, filesize=16.3 K 2024-11-15T04:39:53,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/32cfffec15484fa3a8f56d6952c4c28e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/32cfffec15484fa3a8f56d6952c4c28e 2024-11-15T04:39:53,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/32cfffec15484fa3a8f56d6952c4c28e, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T04:39:53,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/4052a0e9f19845fe96ce7e6a86ac9af7 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4052a0e9f19845fe96ce7e6a86ac9af7 2024-11-15T04:39:53,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4052a0e9f19845fe96ce7e6a86ac9af7, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T04:39:53,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for bffc5fba59ef21ffa1864f468203f7b0 in 701ms, sequenceid=89, compaction requested=true 2024-11-15T04:39:53,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:39:53,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:53,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:39:53,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:53,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:39:53,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:39:53,963 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T04:39:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T04:39:53,978 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T04:39:53,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T04:39:53,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:53,979 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T04:39:53,989 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T04:39:53,991 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/B is initiating minor compaction (all files) 2024-11-15T04:39:53,991 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/B in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:53,992 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/81b305d66b26498696f95f50efeab58f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/968235c733c04d809a374bf985d5cb5f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/44aa9f1365964518a0325db644b30d4e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/50e1c5e4a6194459ace2e56258bb6f23, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/32cfffec15484fa3a8f56d6952c4c28e] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=56.3 K 2024-11-15T04:39:53,995 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 81b305d66b26498696f95f50efeab58f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731645588057 2024-11-15T04:39:53,996 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 71701 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T04:39:53,997 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/A is initiating minor compaction (all files) 2024-11-15T04:39:53,997 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/A in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:53,997 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/68b108ea5d544798b0f0e3388ec42b0c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fde0a0dbf00e407594e6d7199e9d2795, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9ea27444b32b4a84a13004a8cfd7e795, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/ed4d264c193d4aa181b79109192ba905, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/50bd26e121f848f4a42b9aed77cf4814] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=70.0 K 2024-11-15T04:39:53,999 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68b108ea5d544798b0f0e3388ec42b0c, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731645588041 2024-11-15T04:39:53,999 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 968235c733c04d809a374bf985d5cb5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731645588289 2024-11-15T04:39:54,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/fb5168d07fd54b5488e370c84ee8b240 is 50, key is test_row_0/A:col10/1731645593376/Put/seqid=0 2024-11-15T04:39:54,002 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting fde0a0dbf00e407594e6d7199e9d2795, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731645588289 2024-11-15T04:39:54,002 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 44aa9f1365964518a0325db644b30d4e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731645589599 2024-11-15T04:39:54,002 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ea27444b32b4a84a13004a8cfd7e795, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731645589599 2024-11-15T04:39:54,004 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed4d264c193d4aa181b79109192ba905, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731645590344 2024-11-15T04:39:54,006 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50bd26e121f848f4a42b9aed77cf4814, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731645592584 2024-11-15T04:39:54,007 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 50e1c5e4a6194459ace2e56258bb6f23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731645590344 2024-11-15T04:39:54,009 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 32cfffec15484fa3a8f56d6952c4c28e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731645592584 2024-11-15T04:39:54,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741852_1028 (size=12001) 2024-11-15T04:39:54,044 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/fb5168d07fd54b5488e370c84ee8b240 2024-11-15T04:39:54,066 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#A#compaction#16 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:54,067 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/cdb325e6e465437490dd7e0ae96da197 is 50, key is test_row_0/A:col10/1731645593248/Put/seqid=0 2024-11-15T04:39:54,098 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#B#compaction#17 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:54,099 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/0e0518cd48df47f7a5a7f7ed70149904 is 50, key is test_row_0/B:col10/1731645593248/Put/seqid=0 2024-11-15T04:39:54,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:54,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:54,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741853_1029 (size=12173) 2024-11-15T04:39:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/834b2833f26c4ee3b184a8ff069022b3 is 50, key is test_row_0/B:col10/1731645593376/Put/seqid=0 2024-11-15T04:39:54,159 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/cdb325e6e465437490dd7e0ae96da197 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/cdb325e6e465437490dd7e0ae96da197 2024-11-15T04:39:54,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741854_1030 (size=12173) 2024-11-15T04:39:54,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645654162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645654163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645654163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645654166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645654170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,193 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/A of bffc5fba59ef21ffa1864f468203f7b0 into cdb325e6e465437490dd7e0ae96da197(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:54,193 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:54,196 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/A, priority=11, startTime=1731645593953; duration=0sec 2024-11-15T04:39:54,197 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:39:54,197 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:A 2024-11-15T04:39:54,197 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T04:39:54,201 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/0e0518cd48df47f7a5a7f7ed70149904 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0e0518cd48df47f7a5a7f7ed70149904 2024-11-15T04:39:54,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741855_1031 (size=12001) 2024-11-15T04:39:54,210 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 57661 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T04:39:54,210 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/C is initiating minor compaction (all files) 2024-11-15T04:39:54,210 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/C in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:54,210 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/16836675119a4dd38fd7d36b188993d0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8ae82e46226940d4b6df324400f53cc5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/99e9d5f8403b4ef495fc2b09890e638e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/24232a027c744cd0b7ba5157260cdfc7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4052a0e9f19845fe96ce7e6a86ac9af7] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=56.3 K 2024-11-15T04:39:54,212 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16836675119a4dd38fd7d36b188993d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731645588057 2024-11-15T04:39:54,214 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ae82e46226940d4b6df324400f53cc5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731645588289 2024-11-15T04:39:54,216 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99e9d5f8403b4ef495fc2b09890e638e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731645589599 2024-11-15T04:39:54,218 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/B of bffc5fba59ef21ffa1864f468203f7b0 into 0e0518cd48df47f7a5a7f7ed70149904(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:54,218 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:54,218 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/B, priority=11, startTime=1731645593960; duration=0sec 2024-11-15T04:39:54,218 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:54,218 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:B 2024-11-15T04:39:54,222 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24232a027c744cd0b7ba5157260cdfc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731645590344 2024-11-15T04:39:54,223 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4052a0e9f19845fe96ce7e6a86ac9af7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731645592584 2024-11-15T04:39:54,267 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#C#compaction#19 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:54,268 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/94ef9b881cb940078cbd79b04569346b is 50, key is test_row_0/C:col10/1731645593248/Put/seqid=0 2024-11-15T04:39:54,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645654293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645654293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645654293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645654293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741856_1032 (size=12173) 2024-11-15T04:39:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645654303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T04:39:54,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645654514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645654514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,517 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645654514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645654523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645654524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,605 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/834b2833f26c4ee3b184a8ff069022b3 2024-11-15T04:39:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/be51123b6842457abc8f1f2f9c2a8100 is 50, key is test_row_0/C:col10/1731645593376/Put/seqid=0 2024-11-15T04:39:54,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741857_1033 (size=12001) 2024-11-15T04:39:54,657 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/be51123b6842457abc8f1f2f9c2a8100 2024-11-15T04:39:54,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/fb5168d07fd54b5488e370c84ee8b240 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fb5168d07fd54b5488e370c84ee8b240 2024-11-15T04:39:54,690 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fb5168d07fd54b5488e370c84ee8b240, entries=150, sequenceid=112, filesize=11.7 K 2024-11-15T04:39:54,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/834b2833f26c4ee3b184a8ff069022b3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/834b2833f26c4ee3b184a8ff069022b3 2024-11-15T04:39:54,716 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/834b2833f26c4ee3b184a8ff069022b3, entries=150, sequenceid=112, filesize=11.7 K 2024-11-15T04:39:54,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/be51123b6842457abc8f1f2f9c2a8100 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/be51123b6842457abc8f1f2f9c2a8100 2024-11-15T04:39:54,734 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/be51123b6842457abc8f1f2f9c2a8100, entries=150, sequenceid=112, filesize=11.7 K 2024-11-15T04:39:54,737 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for bffc5fba59ef21ffa1864f468203f7b0 in 758ms, sequenceid=112, compaction requested=false 2024-11-15T04:39:54,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:54,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:54,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T04:39:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T04:39:54,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T04:39:54,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3880 sec 2024-11-15T04:39:54,750 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/94ef9b881cb940078cbd79b04569346b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/94ef9b881cb940078cbd79b04569346b 2024-11-15T04:39:54,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4030 sec 2024-11-15T04:39:54,764 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/C of bffc5fba59ef21ffa1864f468203f7b0 into 94ef9b881cb940078cbd79b04569346b(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:54,765 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:54,765 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/C, priority=11, startTime=1731645593960; duration=0sec 2024-11-15T04:39:54,765 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:54,765 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:C 2024-11-15T04:39:54,838 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:39:54,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T04:39:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:54,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/c28c1317512a4fa0912ce38bcfbd7f27 is 50, key is test_row_0/A:col10/1731645594162/Put/seqid=0 2024-11-15T04:39:54,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741858_1034 (size=16781) 2024-11-15T04:39:54,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645654924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645654924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645654926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645654926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:54,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645654926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645655032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645655044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645655044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645655044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645655044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645655245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645655264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645655264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645655264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645655264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/c28c1317512a4fa0912ce38bcfbd7f27 2024-11-15T04:39:55,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/0b5384df360d4c929dbcd528114bceed is 50, key is test_row_0/B:col10/1731645594162/Put/seqid=0 2024-11-15T04:39:55,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741859_1035 (size=12051) 2024-11-15T04:39:55,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/0b5384df360d4c929dbcd528114bceed 2024-11-15T04:39:55,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/31d794a8381d491da4ebd04e6355ba43 is 50, key is test_row_0/C:col10/1731645594162/Put/seqid=0 2024-11-15T04:39:55,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741860_1036 (size=12051) 2024-11-15T04:39:55,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/31d794a8381d491da4ebd04e6355ba43 2024-11-15T04:39:55,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/c28c1317512a4fa0912ce38bcfbd7f27 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c28c1317512a4fa0912ce38bcfbd7f27 2024-11-15T04:39:55,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c28c1317512a4fa0912ce38bcfbd7f27, entries=250, sequenceid=130, filesize=16.4 K 2024-11-15T04:39:55,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/0b5384df360d4c929dbcd528114bceed as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0b5384df360d4c929dbcd528114bceed 2024-11-15T04:39:55,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0b5384df360d4c929dbcd528114bceed, entries=150, sequenceid=130, filesize=11.8 K 2024-11-15T04:39:55,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/31d794a8381d491da4ebd04e6355ba43 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/31d794a8381d491da4ebd04e6355ba43 2024-11-15T04:39:55,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/31d794a8381d491da4ebd04e6355ba43, entries=150, sequenceid=130, filesize=11.8 K 2024-11-15T04:39:55,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=134.18 KB/137400 for bffc5fba59ef21ffa1864f468203f7b0 in 646ms, sequenceid=130, compaction requested=true 2024-11-15T04:39:55,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:55,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:39:55,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:55,485 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:39:55,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:39:55,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:55,486 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:39:55,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:39:55,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:39:55,489 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40955 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:39:55,489 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:39:55,489 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/A is initiating minor compaction (all files) 2024-11-15T04:39:55,489 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/B is initiating minor compaction (all files) 2024-11-15T04:39:55,489 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/A in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:55,489 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/B in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:55,490 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/cdb325e6e465437490dd7e0ae96da197, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fb5168d07fd54b5488e370c84ee8b240, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c28c1317512a4fa0912ce38bcfbd7f27] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=40.0 K 2024-11-15T04:39:55,490 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0e0518cd48df47f7a5a7f7ed70149904, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/834b2833f26c4ee3b184a8ff069022b3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0b5384df360d4c929dbcd528114bceed] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=35.4 K 2024-11-15T04:39:55,491 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e0518cd48df47f7a5a7f7ed70149904, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731645592584 2024-11-15T04:39:55,491 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdb325e6e465437490dd7e0ae96da197, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731645592584 2024-11-15T04:39:55,491 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb5168d07fd54b5488e370c84ee8b240, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731645593309 2024-11-15T04:39:55,492 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 834b2833f26c4ee3b184a8ff069022b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731645593309 2024-11-15T04:39:55,492 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting c28c1317512a4fa0912ce38bcfbd7f27, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731645594160 2024-11-15T04:39:55,492 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5384df360d4c929dbcd528114bceed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731645594162 2024-11-15T04:39:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T04:39:55,493 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:39:55,494 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:39:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T04:39:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T04:39:55,498 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:39:55,500 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:39:55,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:39:55,513 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#B#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:55,514 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/9dcf0080afa8446d8d9621c6a8fe6191 is 50, key is test_row_0/B:col10/1731645594162/Put/seqid=0 2024-11-15T04:39:55,515 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#A#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:55,516 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/9973262c382a4b33b06b26cdb84241d5 is 50, key is test_row_0/A:col10/1731645594162/Put/seqid=0 2024-11-15T04:39:55,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741861_1037 (size=12325) 2024-11-15T04:39:55,548 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/9dcf0080afa8446d8d9621c6a8fe6191 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/9dcf0080afa8446d8d9621c6a8fe6191 2024-11-15T04:39:55,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:55,559 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:39:55,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T04:39:55,567 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/B of bffc5fba59ef21ffa1864f468203f7b0 into 9dcf0080afa8446d8d9621c6a8fe6191(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:55,567 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:55,568 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/B, priority=13, startTime=1731645595485; duration=0sec 2024-11-15T04:39:55,568 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:39:55,568 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:B 2024-11-15T04:39:55,568 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:39:55,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741862_1038 (size=12325) 2024-11-15T04:39:55,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/5cd89e46f9e3499eaecb917757a6aab5 is 50, key is test_row_0/A:col10/1731645594923/Put/seqid=0 2024-11-15T04:39:55,577 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:39:55,577 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/C is initiating minor compaction (all files) 2024-11-15T04:39:55,577 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/C in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:55,577 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/94ef9b881cb940078cbd79b04569346b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/be51123b6842457abc8f1f2f9c2a8100, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/31d794a8381d491da4ebd04e6355ba43] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=35.4 K 2024-11-15T04:39:55,579 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ef9b881cb940078cbd79b04569346b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731645592584 2024-11-15T04:39:55,581 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting be51123b6842457abc8f1f2f9c2a8100, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731645593309 2024-11-15T04:39:55,582 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d794a8381d491da4ebd04e6355ba43, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731645594162 2024-11-15T04:39:55,584 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/9973262c382a4b33b06b26cdb84241d5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9973262c382a4b33b06b26cdb84241d5 2024-11-15T04:39:55,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741863_1039 (size=14541) 2024-11-15T04:39:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T04:39:55,606 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/A of bffc5fba59ef21ffa1864f468203f7b0 into 9973262c382a4b33b06b26cdb84241d5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:55,606 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:55,606 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/A, priority=13, startTime=1731645595485; duration=0sec 2024-11-15T04:39:55,606 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:55,606 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:A 2024-11-15T04:39:55,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645655598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645655601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645655599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,621 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#C#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:55,622 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/c4b81d7b6e6a41199fdff0c3bac51080 is 50, key is test_row_0/C:col10/1731645594162/Put/seqid=0 2024-11-15T04:39:55,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645655616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645655616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741864_1040 (size=12325) 2024-11-15T04:39:55,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T04:39:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:55,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:55,667 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/c4b81d7b6e6a41199fdff0c3bac51080 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c4b81d7b6e6a41199fdff0c3bac51080 2024-11-15T04:39:55,687 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/C of bffc5fba59ef21ffa1864f468203f7b0 into c4b81d7b6e6a41199fdff0c3bac51080(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:55,687 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:55,687 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/C, priority=13, startTime=1731645595486; duration=0sec 2024-11-15T04:39:55,688 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:55,688 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:C 2024-11-15T04:39:55,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645655723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645655723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645655723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645655743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645655743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T04:39:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:55,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:55,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:55,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T04:39:55,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645655934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645655933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645655943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645655953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645655963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:55,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T04:39:55,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:55,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:55,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:55,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:55,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:55,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/5cd89e46f9e3499eaecb917757a6aab5 2024-11-15T04:39:56,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/3a61d04f5c114ff5997b8c70e8a803cc is 50, key is test_row_0/B:col10/1731645594923/Put/seqid=0 2024-11-15T04:39:56,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741865_1041 (size=12151) 2024-11-15T04:39:56,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/3a61d04f5c114ff5997b8c70e8a803cc 2024-11-15T04:39:56,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/5fdbb2d7ca444082b144160180719e8c is 50, key is test_row_0/C:col10/1731645594923/Put/seqid=0 2024-11-15T04:39:56,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741866_1042 (size=12151) 2024-11-15T04:39:56,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/5fdbb2d7ca444082b144160180719e8c 2024-11-15T04:39:56,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/5cd89e46f9e3499eaecb917757a6aab5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/5cd89e46f9e3499eaecb917757a6aab5 2024-11-15T04:39:56,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/5cd89e46f9e3499eaecb917757a6aab5, entries=200, sequenceid=154, filesize=14.2 K 2024-11-15T04:39:56,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T04:39:56,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/3a61d04f5c114ff5997b8c70e8a803cc as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/3a61d04f5c114ff5997b8c70e8a803cc 2024-11-15T04:39:56,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T04:39:56,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:56,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/3a61d04f5c114ff5997b8c70e8a803cc, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T04:39:56,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/5fdbb2d7ca444082b144160180719e8c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/5fdbb2d7ca444082b144160180719e8c 2024-11-15T04:39:56,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/5fdbb2d7ca444082b144160180719e8c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-15T04:39:56,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for bffc5fba59ef21ffa1864f468203f7b0 in 595ms, sequenceid=154, compaction requested=false 2024-11-15T04:39:56,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:56,256 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:39:56,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T04:39:56,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/6f809a9f03c245b8bdd7fe41af7a6ac8 is 50, key is test_row_0/A:col10/1731645596256/Put/seqid=0 2024-11-15T04:39:56,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741867_1043 (size=12151) 2024-11-15T04:39:56,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/6f809a9f03c245b8bdd7fe41af7a6ac8 2024-11-15T04:39:56,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T04:39:56,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:56,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/410e280d55834e809489acb3f48f8984 is 50, key is test_row_0/B:col10/1731645596256/Put/seqid=0 2024-11-15T04:39:56,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741868_1044 (size=12151) 2024-11-15T04:39:56,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645656299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645656301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645656302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645656312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645656313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645656423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645656423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645656424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645656424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645656424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T04:39:56,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:56,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T04:39:56,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:56,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T04:39:56,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645656633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645656634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645656634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645656634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645656634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/410e280d55834e809489acb3f48f8984 2024-11-15T04:39:56,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/eec0d4251cf444c1b86b7a5a030ff7fa is 50, key is test_row_0/C:col10/1731645596256/Put/seqid=0 2024-11-15T04:39:56,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T04:39:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:56,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741869_1045 (size=12151) 2024-11-15T04:39:56,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/eec0d4251cf444c1b86b7a5a030ff7fa 2024-11-15T04:39:56,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/6f809a9f03c245b8bdd7fe41af7a6ac8 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6f809a9f03c245b8bdd7fe41af7a6ac8 2024-11-15T04:39:56,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6f809a9f03c245b8bdd7fe41af7a6ac8, entries=150, sequenceid=172, filesize=11.9 K 2024-11-15T04:39:56,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/410e280d55834e809489acb3f48f8984 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/410e280d55834e809489acb3f48f8984 2024-11-15T04:39:56,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/410e280d55834e809489acb3f48f8984, entries=150, sequenceid=172, filesize=11.9 K 2024-11-15T04:39:56,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/eec0d4251cf444c1b86b7a5a030ff7fa as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/eec0d4251cf444c1b86b7a5a030ff7fa 2024-11-15T04:39:56,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/eec0d4251cf444c1b86b7a5a030ff7fa, entries=150, sequenceid=172, filesize=11.9 K 2024-11-15T04:39:56,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for bffc5fba59ef21ffa1864f468203f7b0 in 570ms, sequenceid=172, compaction requested=true 2024-11-15T04:39:56,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:56,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:39:56,827 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:39:56,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:56,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:39:56,828 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:39:56,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:56,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:39:56,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:39:56,829 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39017 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:39:56,830 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/A is initiating minor compaction (all files) 2024-11-15T04:39:56,830 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/A in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,830 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9973262c382a4b33b06b26cdb84241d5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/5cd89e46f9e3499eaecb917757a6aab5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6f809a9f03c245b8bdd7fe41af7a6ac8] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=38.1 K 2024-11-15T04:39:56,830 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:39:56,830 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/B is initiating minor compaction (all files) 2024-11-15T04:39:56,831 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/B in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,831 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/9dcf0080afa8446d8d9621c6a8fe6191, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/3a61d04f5c114ff5997b8c70e8a803cc, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/410e280d55834e809489acb3f48f8984] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=35.8 K 2024-11-15T04:39:56,831 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9973262c382a4b33b06b26cdb84241d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731645594162 2024-11-15T04:39:56,832 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dcf0080afa8446d8d9621c6a8fe6191, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731645594162 2024-11-15T04:39:56,832 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cd89e46f9e3499eaecb917757a6aab5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731645594922 2024-11-15T04:39:56,832 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a61d04f5c114ff5997b8c70e8a803cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731645594922 2024-11-15T04:39:56,833 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f809a9f03c245b8bdd7fe41af7a6ac8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731645596255 2024-11-15T04:39:56,833 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 410e280d55834e809489acb3f48f8984, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731645596255 2024-11-15T04:39:56,869 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#B#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:56,870 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/9539b51d2c3546d2abb5f5d6b735ccf9 is 50, key is test_row_0/B:col10/1731645596256/Put/seqid=0 2024-11-15T04:39:56,885 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#A#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:56,886 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/6fdead09714b42e484e4eeecbd85cec1 is 50, key is test_row_0/A:col10/1731645596256/Put/seqid=0 2024-11-15T04:39:56,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741870_1046 (size=12527) 2024-11-15T04:39:56,910 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/9539b51d2c3546d2abb5f5d6b735ccf9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/9539b51d2c3546d2abb5f5d6b735ccf9 2024-11-15T04:39:56,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T04:39:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,912 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T04:39:56,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741871_1047 (size=12527) 2024-11-15T04:39:56,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/828780f458a1471f8d603905f1925ea1 is 50, key is test_row_0/A:col10/1731645596307/Put/seqid=0 2024-11-15T04:39:56,931 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/B of bffc5fba59ef21ffa1864f468203f7b0 into 9539b51d2c3546d2abb5f5d6b735ccf9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:56,931 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:56,931 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/B, priority=13, startTime=1731645596828; duration=0sec 2024-11-15T04:39:56,931 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:39:56,931 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:B 2024-11-15T04:39:56,931 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:39:56,934 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:39:56,934 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/C is initiating minor compaction (all files) 2024-11-15T04:39:56,934 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/C in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:56,934 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c4b81d7b6e6a41199fdff0c3bac51080, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/5fdbb2d7ca444082b144160180719e8c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/eec0d4251cf444c1b86b7a5a030ff7fa] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=35.8 K 2024-11-15T04:39:56,935 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c4b81d7b6e6a41199fdff0c3bac51080, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731645594162 2024-11-15T04:39:56,936 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fdbb2d7ca444082b144160180719e8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731645594922 2024-11-15T04:39:56,936 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting eec0d4251cf444c1b86b7a5a030ff7fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731645596255 2024-11-15T04:39:56,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:56,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741872_1048 (size=12151) 2024-11-15T04:39:56,962 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#C#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:56,962 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/ba4c5a740e3648648e26f9d9b05ad1fa is 50, key is test_row_0/C:col10/1731645596256/Put/seqid=0 2024-11-15T04:39:56,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645656967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645656971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645656972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645656974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:56,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645656971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:56,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:56,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741873_1049 (size=12527) 2024-11-15T04:39:56,996 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/ba4c5a740e3648648e26f9d9b05ad1fa as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/ba4c5a740e3648648e26f9d9b05ad1fa 2024-11-15T04:39:57,005 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/C of bffc5fba59ef21ffa1864f468203f7b0 into ba4c5a740e3648648e26f9d9b05ad1fa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:57,005 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:57,006 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/C, priority=13, startTime=1731645596828; duration=0sec 2024-11-15T04:39:57,006 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:57,006 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:C 2024-11-15T04:39:57,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645657084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645657084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645657084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645657084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645657093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645657293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645657293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645657294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645657294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645657303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,335 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/6fdead09714b42e484e4eeecbd85cec1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6fdead09714b42e484e4eeecbd85cec1 2024-11-15T04:39:57,346 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/A of bffc5fba59ef21ffa1864f468203f7b0 into 6fdead09714b42e484e4eeecbd85cec1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:57,346 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:57,346 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/A, priority=13, startTime=1731645596827; duration=0sec 2024-11-15T04:39:57,346 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:57,347 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:A 2024-11-15T04:39:57,353 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/828780f458a1471f8d603905f1925ea1 2024-11-15T04:39:57,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/5edc23d0f04746bebc5c1cf8829b70ab is 50, key is test_row_0/B:col10/1731645596307/Put/seqid=0 2024-11-15T04:39:57,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741874_1050 (size=12151) 2024-11-15T04:39:57,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645657603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645657604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645657605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645657604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645657615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:57,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:57,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T04:39:57,802 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/5edc23d0f04746bebc5c1cf8829b70ab 2024-11-15T04:39:57,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/4a955837215a482983f4915dfdb7b15c is 50, key is test_row_0/C:col10/1731645596307/Put/seqid=0 2024-11-15T04:39:57,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741875_1051 (size=12151) 2024-11-15T04:39:58,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645658112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:58,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645658114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:58,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:58,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:58,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645658123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:58,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645658123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:58,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645658123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:58,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:58,223 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/4a955837215a482983f4915dfdb7b15c 2024-11-15T04:39:58,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/828780f458a1471f8d603905f1925ea1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/828780f458a1471f8d603905f1925ea1 2024-11-15T04:39:58,242 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/828780f458a1471f8d603905f1925ea1, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T04:39:58,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/5edc23d0f04746bebc5c1cf8829b70ab as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/5edc23d0f04746bebc5c1cf8829b70ab 2024-11-15T04:39:58,256 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/5edc23d0f04746bebc5c1cf8829b70ab, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T04:39:58,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/4a955837215a482983f4915dfdb7b15c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4a955837215a482983f4915dfdb7b15c 2024-11-15T04:39:58,267 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4a955837215a482983f4915dfdb7b15c, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T04:39:58,269 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for bffc5fba59ef21ffa1864f468203f7b0 in 1357ms, sequenceid=194, compaction requested=false 2024-11-15T04:39:58,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:58,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:58,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T04:39:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T04:39:58,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T04:39:58,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7740 sec 2024-11-15T04:39:58,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.7870 sec 2024-11-15T04:39:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:39:59,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T04:39:59,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/3d5dc439a5bc4f4bad3c218dade34bde is 50, key is test_row_0/A:col10/1731645599124/Put/seqid=0 2024-11-15T04:39:59,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741876_1052 (size=14541) 2024-11-15T04:39:59,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/3d5dc439a5bc4f4bad3c218dade34bde 2024-11-15T04:39:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645659166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645659167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645659175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645659178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/ea67e326075e41bfa84a4a4157ab83b3 is 50, key is test_row_0/B:col10/1731645599124/Put/seqid=0 2024-11-15T04:39:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645659182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741877_1053 (size=12151) 2024-11-15T04:39:59,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/ea67e326075e41bfa84a4a4157ab83b3 2024-11-15T04:39:59,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/f3f2962867a64270ac2b52e3946d359b is 50, key is test_row_0/C:col10/1731645599124/Put/seqid=0 2024-11-15T04:39:59,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741878_1054 (size=12151) 2024-11-15T04:39:59,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645659282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645659283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645659293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645659293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645659293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645659493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645659493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645659503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645659503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645659503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/f3f2962867a64270ac2b52e3946d359b 2024-11-15T04:39:59,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/3d5dc439a5bc4f4bad3c218dade34bde as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/3d5dc439a5bc4f4bad3c218dade34bde 2024-11-15T04:39:59,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/3d5dc439a5bc4f4bad3c218dade34bde, entries=200, sequenceid=213, filesize=14.2 K 2024-11-15T04:39:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/ea67e326075e41bfa84a4a4157ab83b3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ea67e326075e41bfa84a4a4157ab83b3 2024-11-15T04:39:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T04:39:59,663 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:39:59,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ea67e326075e41bfa84a4a4157ab83b3, entries=150, sequenceid=213, filesize=11.9 K 2024-11-15T04:39:59,664 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:39:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T04:39:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T04:39:59,667 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:39:59,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/f3f2962867a64270ac2b52e3946d359b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/f3f2962867a64270ac2b52e3946d359b 2024-11-15T04:39:59,672 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:39:59,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:39:59,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/f3f2962867a64270ac2b52e3946d359b, entries=150, sequenceid=213, filesize=11.9 K 2024-11-15T04:39:59,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for bffc5fba59ef21ffa1864f468203f7b0 in 558ms, sequenceid=213, compaction requested=true 2024-11-15T04:39:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:39:59,683 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:39:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:59,683 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:39:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:39:59,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:59,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:39:59,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:39:59,685 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:39:59,685 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/A is initiating minor compaction (all files) 2024-11-15T04:39:59,685 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/A in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:59,685 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6fdead09714b42e484e4eeecbd85cec1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/828780f458a1471f8d603905f1925ea1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/3d5dc439a5bc4f4bad3c218dade34bde] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=38.3 K 2024-11-15T04:39:59,685 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:39:59,685 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/B is initiating minor compaction (all files) 2024-11-15T04:39:59,686 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/B in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:59,686 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/9539b51d2c3546d2abb5f5d6b735ccf9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/5edc23d0f04746bebc5c1cf8829b70ab, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ea67e326075e41bfa84a4a4157ab83b3] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=36.0 K 2024-11-15T04:39:59,687 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9539b51d2c3546d2abb5f5d6b735ccf9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731645596255 2024-11-15T04:39:59,687 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fdead09714b42e484e4eeecbd85cec1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731645596255 2024-11-15T04:39:59,688 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 5edc23d0f04746bebc5c1cf8829b70ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731645596297 2024-11-15T04:39:59,689 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 828780f458a1471f8d603905f1925ea1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731645596297 2024-11-15T04:39:59,689 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting ea67e326075e41bfa84a4a4157ab83b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731645596972 2024-11-15T04:39:59,690 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d5dc439a5bc4f4bad3c218dade34bde, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731645596968 2024-11-15T04:39:59,709 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:59,710 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/009e47bd6fcb4e0f80edbe000c0252e9 is 50, key is test_row_0/B:col10/1731645599124/Put/seqid=0 2024-11-15T04:39:59,717 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#A#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:59,717 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/6b37c46c720f4e3497cd838e73f40270 is 50, key is test_row_0/A:col10/1731645599124/Put/seqid=0 2024-11-15T04:39:59,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741879_1055 (size=12629) 2024-11-15T04:39:59,758 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/009e47bd6fcb4e0f80edbe000c0252e9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/009e47bd6fcb4e0f80edbe000c0252e9 2024-11-15T04:39:59,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741880_1056 (size=12629) 2024-11-15T04:39:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T04:39:59,795 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/B of bffc5fba59ef21ffa1864f468203f7b0 into 009e47bd6fcb4e0f80edbe000c0252e9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:59,795 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:59,795 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/B, priority=13, startTime=1731645599683; duration=0sec 2024-11-15T04:39:59,795 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:39:59,795 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:B 2024-11-15T04:39:59,795 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:39:59,800 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:39:59,800 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/C is initiating minor compaction (all files) 2024-11-15T04:39:59,800 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/C in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:59,803 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/ba4c5a740e3648648e26f9d9b05ad1fa, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4a955837215a482983f4915dfdb7b15c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/f3f2962867a64270ac2b52e3946d359b] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=36.0 K 2024-11-15T04:39:59,804 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting ba4c5a740e3648648e26f9d9b05ad1fa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731645596255 2024-11-15T04:39:59,805 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a955837215a482983f4915dfdb7b15c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731645596297 2024-11-15T04:39:59,805 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f2962867a64270ac2b52e3946d359b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731645596972 2024-11-15T04:39:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:39:59,806 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:39:59,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T04:39:59,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/e3c7b526f636406cb71414ebcf70d151 is 50, key is test_row_0/A:col10/1731645599804/Put/seqid=0 2024-11-15T04:39:59,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T04:39:59,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:59,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:59,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:59,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:59,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645659831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645659831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645659833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645659841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645659841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741881_1057 (size=14541) 2024-11-15T04:39:59,862 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:39:59,862 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/cf23eea2544a4226a168814ecca41546 is 50, key is test_row_0/C:col10/1731645599124/Put/seqid=0 2024-11-15T04:39:59,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741882_1058 (size=12629) 2024-11-15T04:39:59,919 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/cf23eea2544a4226a168814ecca41546 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/cf23eea2544a4226a168814ecca41546 2024-11-15T04:39:59,937 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/C of bffc5fba59ef21ffa1864f468203f7b0 into cf23eea2544a4226a168814ecca41546(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:39:59,937 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:39:59,937 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/C, priority=13, startTime=1731645599684; duration=0sec 2024-11-15T04:39:59,937 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:39:59,937 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:C 2024-11-15T04:39:59,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645659954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645659954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645659954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645659953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:39:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645659954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:39:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:39:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:39:59,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T04:39:59,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:59,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:39:59,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:39:59,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:59,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:39:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T04:40:00,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T04:40:00,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645660174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645660174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645660174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645660174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:00,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645660174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:00,189 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/6b37c46c720f4e3497cd838e73f40270 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6b37c46c720f4e3497cd838e73f40270 2024-11-15T04:40:00,216 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/A of bffc5fba59ef21ffa1864f468203f7b0 into 6b37c46c720f4e3497cd838e73f40270(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:00,217 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:00,217 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/A, priority=13, startTime=1731645599683; duration=0sec 2024-11-15T04:40:00,217 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:00,217 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:A 2024-11-15T04:40:00,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/e3c7b526f636406cb71414ebcf70d151 2024-11-15T04:40:00,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/7bb25ccc35414e4ba5cc123ac816d6b3 is 50, key is test_row_0/B:col10/1731645599804/Put/seqid=0 2024-11-15T04:40:00,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T04:40:00,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:00,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T04:40:00,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741883_1059 (size=12151) 2024-11-15T04:40:00,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/7bb25ccc35414e4ba5cc123ac816d6b3 2024-11-15T04:40:00,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/d137d407344f43e682c471d84551dad1 is 50, key is test_row_0/C:col10/1731645599804/Put/seqid=0 2024-11-15T04:40:00,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741884_1060 (size=12151) 2024-11-15T04:40:00,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T04:40:00,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:00,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645660482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645660482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:00,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645660483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645660483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:00,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645660492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:00,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:00,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T04:40:00,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:00,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T04:40:00,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:00,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:00,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/d137d407344f43e682c471d84551dad1 2024-11-15T04:40:00,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/e3c7b526f636406cb71414ebcf70d151 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/e3c7b526f636406cb71414ebcf70d151 2024-11-15T04:40:00,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/e3c7b526f636406cb71414ebcf70d151, entries=200, sequenceid=236, filesize=14.2 K 2024-11-15T04:40:00,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/7bb25ccc35414e4ba5cc123ac816d6b3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7bb25ccc35414e4ba5cc123ac816d6b3 2024-11-15T04:40:00,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7bb25ccc35414e4ba5cc123ac816d6b3, entries=150, sequenceid=236, filesize=11.9 K 2024-11-15T04:40:00,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/d137d407344f43e682c471d84551dad1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d137d407344f43e682c471d84551dad1 2024-11-15T04:40:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T04:40:00,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d137d407344f43e682c471d84551dad1, entries=150, sequenceid=236, filesize=11.9 K 2024-11-15T04:40:00,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for bffc5fba59ef21ffa1864f468203f7b0 in 1009ms, sequenceid=236, compaction requested=false 2024-11-15T04:40:00,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:00,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T04:40:00,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:00,911 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T04:40:00,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/44ea9e74d3314235b9d9f98627cd2fd5 is 50, key is test_row_0/A:col10/1731645599838/Put/seqid=0 2024-11-15T04:40:00,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741885_1061 (size=12151) 2024-11-15T04:40:00,970 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/44ea9e74d3314235b9d9f98627cd2fd5 2024-11-15T04:40:00,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/ac4af21b244c4a66b2283de92f2c53ce is 50, key is test_row_0/B:col10/1731645599838/Put/seqid=0 2024-11-15T04:40:00,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:01,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741886_1062 (size=12151) 2024-11-15T04:40:01,022 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/ac4af21b244c4a66b2283de92f2c53ce 2024-11-15T04:40:01,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/30e3050e1c5d4a0abbeea502d9ab5b65 is 50, key is test_row_0/C:col10/1731645599838/Put/seqid=0 2024-11-15T04:40:01,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645661038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645661040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741887_1063 (size=12151) 2024-11-15T04:40:01,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645661044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645661048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,053 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/30e3050e1c5d4a0abbeea502d9ab5b65 2024-11-15T04:40:01,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645661049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/44ea9e74d3314235b9d9f98627cd2fd5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/44ea9e74d3314235b9d9f98627cd2fd5 2024-11-15T04:40:01,080 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/44ea9e74d3314235b9d9f98627cd2fd5, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T04:40:01,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/ac4af21b244c4a66b2283de92f2c53ce as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ac4af21b244c4a66b2283de92f2c53ce 2024-11-15T04:40:01,091 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ac4af21b244c4a66b2283de92f2c53ce, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T04:40:01,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/30e3050e1c5d4a0abbeea502d9ab5b65 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/30e3050e1c5d4a0abbeea502d9ab5b65 2024-11-15T04:40:01,123 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/30e3050e1c5d4a0abbeea502d9ab5b65, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T04:40:01,124 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for bffc5fba59ef21ffa1864f468203f7b0 in 213ms, sequenceid=253, compaction requested=true 2024-11-15T04:40:01,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:01,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:01,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T04:40:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T04:40:01,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T04:40:01,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4540 sec 2024-11-15T04:40:01,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4690 sec 2024-11-15T04:40:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:01,155 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:01,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T04:40:01,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/0448b5f3ed994c11904ecbb053f8eea2 is 50, key is test_row_0/A:col10/1731645601154/Put/seqid=0 2024-11-15T04:40:01,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741888_1064 (size=12301) 2024-11-15T04:40:01,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/0448b5f3ed994c11904ecbb053f8eea2 2024-11-15T04:40:01,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/17d0c85f2cc74a04bc237bb87ee4d65a is 50, key is test_row_0/B:col10/1731645601154/Put/seqid=0 2024-11-15T04:40:01,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741889_1065 (size=12301) 2024-11-15T04:40:01,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645661210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645661209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645661214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645661215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645661215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645661323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645661323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645661323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645661333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645661333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645661533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645661533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645661533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645661543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645661544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/17d0c85f2cc74a04bc237bb87ee4d65a 2024-11-15T04:40:01,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/fbe1a4c3fcc64b68bd4dcef6b18b6b5c is 50, key is test_row_0/C:col10/1731645601154/Put/seqid=0 2024-11-15T04:40:01,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741890_1066 (size=12301) 2024-11-15T04:40:01,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/fbe1a4c3fcc64b68bd4dcef6b18b6b5c 2024-11-15T04:40:01,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/0448b5f3ed994c11904ecbb053f8eea2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/0448b5f3ed994c11904ecbb053f8eea2 2024-11-15T04:40:01,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/0448b5f3ed994c11904ecbb053f8eea2, entries=150, sequenceid=276, filesize=12.0 K 2024-11-15T04:40:01,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/17d0c85f2cc74a04bc237bb87ee4d65a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/17d0c85f2cc74a04bc237bb87ee4d65a 2024-11-15T04:40:01,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/17d0c85f2cc74a04bc237bb87ee4d65a, entries=150, sequenceid=276, filesize=12.0 K 2024-11-15T04:40:01,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/fbe1a4c3fcc64b68bd4dcef6b18b6b5c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/fbe1a4c3fcc64b68bd4dcef6b18b6b5c 2024-11-15T04:40:01,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/fbe1a4c3fcc64b68bd4dcef6b18b6b5c, entries=150, sequenceid=276, filesize=12.0 K 2024-11-15T04:40:01,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for bffc5fba59ef21ffa1864f468203f7b0 in 527ms, sequenceid=276, compaction requested=true 2024-11-15T04:40:01,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:01,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:01,682 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:01,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:01,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:01,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:01,682 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:01,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:01,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:01,684 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51622 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:01,684 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/A is initiating minor compaction (all files) 2024-11-15T04:40:01,684 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/A in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:01,684 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6b37c46c720f4e3497cd838e73f40270, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/e3c7b526f636406cb71414ebcf70d151, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/44ea9e74d3314235b9d9f98627cd2fd5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/0448b5f3ed994c11904ecbb053f8eea2] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=50.4 K 2024-11-15T04:40:01,685 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:01,685 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b37c46c720f4e3497cd838e73f40270, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731645596972 2024-11-15T04:40:01,685 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/B is initiating minor compaction (all files) 2024-11-15T04:40:01,685 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/B in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:01,685 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/009e47bd6fcb4e0f80edbe000c0252e9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7bb25ccc35414e4ba5cc123ac816d6b3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ac4af21b244c4a66b2283de92f2c53ce, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/17d0c85f2cc74a04bc237bb87ee4d65a] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=48.1 K 2024-11-15T04:40:01,685 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c7b526f636406cb71414ebcf70d151, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731645599164 2024-11-15T04:40:01,686 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 009e47bd6fcb4e0f80edbe000c0252e9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731645596972 2024-11-15T04:40:01,687 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ea9e74d3314235b9d9f98627cd2fd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731645599829 2024-11-15T04:40:01,687 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bb25ccc35414e4ba5cc123ac816d6b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731645599171 2024-11-15T04:40:01,687 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0448b5f3ed994c11904ecbb053f8eea2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731645601041 2024-11-15T04:40:01,688 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4af21b244c4a66b2283de92f2c53ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731645599829 2024-11-15T04:40:01,689 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 17d0c85f2cc74a04bc237bb87ee4d65a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731645601041 2024-11-15T04:40:01,725 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#A#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:01,726 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/29f3890b42f7462b8acf34ab5088e123 is 50, key is test_row_0/A:col10/1731645601154/Put/seqid=0 2024-11-15T04:40:01,730 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#B#compaction#55 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:01,731 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/b447e70751c745509c92cbe5657a913f is 50, key is test_row_0/B:col10/1731645601154/Put/seqid=0 2024-11-15T04:40:01,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741892_1068 (size=12915) 2024-11-15T04:40:01,772 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/b447e70751c745509c92cbe5657a913f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/b447e70751c745509c92cbe5657a913f 2024-11-15T04:40:01,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741891_1067 (size=12915) 2024-11-15T04:40:01,786 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/B of bffc5fba59ef21ffa1864f468203f7b0 into b447e70751c745509c92cbe5657a913f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:01,786 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:01,786 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/B, priority=12, startTime=1731645601682; duration=0sec 2024-11-15T04:40:01,787 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:01,787 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:B 2024-11-15T04:40:01,787 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:01,790 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:01,790 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/C is initiating minor compaction (all files) 2024-11-15T04:40:01,790 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/C in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:01,791 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/cf23eea2544a4226a168814ecca41546, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d137d407344f43e682c471d84551dad1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/30e3050e1c5d4a0abbeea502d9ab5b65, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/fbe1a4c3fcc64b68bd4dcef6b18b6b5c] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=48.1 K 2024-11-15T04:40:01,792 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/29f3890b42f7462b8acf34ab5088e123 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/29f3890b42f7462b8acf34ab5088e123 2024-11-15T04:40:01,792 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting cf23eea2544a4226a168814ecca41546, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731645596972 2024-11-15T04:40:01,793 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting d137d407344f43e682c471d84551dad1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731645599171 2024-11-15T04:40:01,793 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 30e3050e1c5d4a0abbeea502d9ab5b65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731645599829 2024-11-15T04:40:01,794 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting fbe1a4c3fcc64b68bd4dcef6b18b6b5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731645601041 2024-11-15T04:40:01,801 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/A of bffc5fba59ef21ffa1864f468203f7b0 into 29f3890b42f7462b8acf34ab5088e123(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:01,801 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:01,801 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/A, priority=12, startTime=1731645601682; duration=0sec 2024-11-15T04:40:01,801 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:01,801 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:A 2024-11-15T04:40:01,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T04:40:01,823 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:01,824 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T04:40:01,828 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:01,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T04:40:01,833 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:01,834 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/c38726c09e5a42538a085d6e99769db3 is 50, key is test_row_0/C:col10/1731645601154/Put/seqid=0 2024-11-15T04:40:01,836 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:01,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:01,845 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:01,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T04:40:01,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741893_1069 (size=12915) 2024-11-15T04:40:01,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/07077e8db916429b8a31bfb679fce3e3 is 50, key is test_row_0/A:col10/1731645601845/Put/seqid=0 2024-11-15T04:40:01,861 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/c38726c09e5a42538a085d6e99769db3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c38726c09e5a42538a085d6e99769db3 2024-11-15T04:40:01,873 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/C of bffc5fba59ef21ffa1864f468203f7b0 into c38726c09e5a42538a085d6e99769db3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:01,873 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:01,874 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/C, priority=12, startTime=1731645601682; duration=0sec 2024-11-15T04:40:01,874 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:01,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741894_1070 (size=14741) 2024-11-15T04:40:01,874 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:C 2024-11-15T04:40:01,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645661881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645661887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645661888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645661889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645661890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T04:40:01,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T04:40:01,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:01,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:01,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:01,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:01,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:01,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645661993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645662003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645662003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645662003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645662003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T04:40:02,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T04:40:02,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:02,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:02,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:02,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:02,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:02,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645662203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645662213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645662215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645662215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645662215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/07077e8db916429b8a31bfb679fce3e3 2024-11-15T04:40:02,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T04:40:02,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/a022db5b1cb845cebd9f68418b75a222 is 50, key is test_row_0/B:col10/1731645601845/Put/seqid=0 2024-11-15T04:40:02,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:02,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:02,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:02,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:02,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:02,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741895_1071 (size=12301) 2024-11-15T04:40:02,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/a022db5b1cb845cebd9f68418b75a222 2024-11-15T04:40:02,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/23f568fb1451467f82212898dca53761 is 50, key is test_row_0/C:col10/1731645601845/Put/seqid=0 2024-11-15T04:40:02,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741896_1072 (size=12301) 2024-11-15T04:40:02,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/23f568fb1451467f82212898dca53761 2024-11-15T04:40:02,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/07077e8db916429b8a31bfb679fce3e3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/07077e8db916429b8a31bfb679fce3e3 2024-11-15T04:40:02,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/07077e8db916429b8a31bfb679fce3e3, entries=200, sequenceid=293, filesize=14.4 K 2024-11-15T04:40:02,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/a022db5b1cb845cebd9f68418b75a222 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/a022db5b1cb845cebd9f68418b75a222 2024-11-15T04:40:02,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/a022db5b1cb845cebd9f68418b75a222, entries=150, sequenceid=293, filesize=12.0 K 2024-11-15T04:40:02,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/23f568fb1451467f82212898dca53761 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/23f568fb1451467f82212898dca53761 2024-11-15T04:40:02,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/23f568fb1451467f82212898dca53761, entries=150, sequenceid=293, filesize=12.0 K 2024-11-15T04:40:02,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for bffc5fba59ef21ffa1864f468203f7b0 in 606ms, sequenceid=293, compaction requested=false 2024-11-15T04:40:02,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T04:40:02,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T04:40:02,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:02,475 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T04:40:02,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/941c215d519b48e7950bbca2e77a2956 is 50, key is test_row_0/A:col10/1731645601886/Put/seqid=0 2024-11-15T04:40:02,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741897_1073 (size=12301) 2024-11-15T04:40:02,516 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/941c215d519b48e7950bbca2e77a2956 2024-11-15T04:40:02,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/f8634dbbc53042678dcae8ef7b95c6ec is 50, key is test_row_0/B:col10/1731645601886/Put/seqid=0 2024-11-15T04:40:02,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:02,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:02,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741898_1074 (size=12301) 2024-11-15T04:40:02,563 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/f8634dbbc53042678dcae8ef7b95c6ec 2024-11-15T04:40:02,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645662548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645662550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645662550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645662565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645662564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/c7766476fa654372a720318f794539bf is 50, key is test_row_0/C:col10/1731645601886/Put/seqid=0 2024-11-15T04:40:02,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741899_1075 (size=12301) 2024-11-15T04:40:02,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645662673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645662673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645662673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645662683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645662683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645662883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645662883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645662884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645662893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645662893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T04:40:03,033 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/c7766476fa654372a720318f794539bf 2024-11-15T04:40:03,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/941c215d519b48e7950bbca2e77a2956 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/941c215d519b48e7950bbca2e77a2956 2024-11-15T04:40:03,060 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/941c215d519b48e7950bbca2e77a2956, entries=150, sequenceid=316, filesize=12.0 K 2024-11-15T04:40:03,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/f8634dbbc53042678dcae8ef7b95c6ec as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f8634dbbc53042678dcae8ef7b95c6ec 2024-11-15T04:40:03,070 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f8634dbbc53042678dcae8ef7b95c6ec, entries=150, sequenceid=316, filesize=12.0 K 2024-11-15T04:40:03,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/c7766476fa654372a720318f794539bf as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c7766476fa654372a720318f794539bf 2024-11-15T04:40:03,088 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c7766476fa654372a720318f794539bf, entries=150, sequenceid=316, filesize=12.0 K 2024-11-15T04:40:03,097 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for bffc5fba59ef21ffa1864f468203f7b0 in 621ms, sequenceid=316, compaction requested=true 2024-11-15T04:40:03,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:03,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:03,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T04:40:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T04:40:03,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T04:40:03,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2640 sec 2024-11-15T04:40:03,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2840 sec 2024-11-15T04:40:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:03,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T04:40:03,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/b1b2d645438d440cb0dcb1cc0cc05965 is 50, key is test_row_0/A:col10/1731645602547/Put/seqid=0 2024-11-15T04:40:03,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741900_1076 (size=14741) 2024-11-15T04:40:03,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/b1b2d645438d440cb0dcb1cc0cc05965 2024-11-15T04:40:03,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645663243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/2f6712c21246456d99c65ee336051d75 is 50, key is test_row_0/B:col10/1731645602547/Put/seqid=0 2024-11-15T04:40:03,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645663243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645663248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645663250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645663254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741901_1077 (size=12301) 2024-11-15T04:40:03,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/2f6712c21246456d99c65ee336051d75 2024-11-15T04:40:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/26b492cd82914e778dba605983c6fe2e is 50, key is test_row_0/C:col10/1731645602547/Put/seqid=0 2024-11-15T04:40:03,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741902_1078 (size=12301) 2024-11-15T04:40:03,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/26b492cd82914e778dba605983c6fe2e 2024-11-15T04:40:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/b1b2d645438d440cb0dcb1cc0cc05965 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/b1b2d645438d440cb0dcb1cc0cc05965 2024-11-15T04:40:03,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/b1b2d645438d440cb0dcb1cc0cc05965, entries=200, sequenceid=333, filesize=14.4 K 2024-11-15T04:40:03,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/2f6712c21246456d99c65ee336051d75 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2f6712c21246456d99c65ee336051d75 2024-11-15T04:40:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645663362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645663363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645663363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645663363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2f6712c21246456d99c65ee336051d75, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T04:40:03,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/26b492cd82914e778dba605983c6fe2e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/26b492cd82914e778dba605983c6fe2e 2024-11-15T04:40:03,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645663373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/26b492cd82914e778dba605983c6fe2e, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T04:40:03,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for bffc5fba59ef21ffa1864f468203f7b0 in 184ms, sequenceid=333, compaction requested=true 2024-11-15T04:40:03,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:03,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:03,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:03,383 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:03,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:03,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:03,383 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:03,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:03,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:03,384 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:03,384 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/B is initiating minor compaction (all files) 2024-11-15T04:40:03,385 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/B in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:03,385 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/b447e70751c745509c92cbe5657a913f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/a022db5b1cb845cebd9f68418b75a222, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f8634dbbc53042678dcae8ef7b95c6ec, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2f6712c21246456d99c65ee336051d75] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=48.7 K 2024-11-15T04:40:03,385 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:03,385 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/A is initiating minor compaction (all files) 2024-11-15T04:40:03,385 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting b447e70751c745509c92cbe5657a913f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731645601041 2024-11-15T04:40:03,385 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/A in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:03,386 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting a022db5b1cb845cebd9f68418b75a222, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731645601172 2024-11-15T04:40:03,386 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting f8634dbbc53042678dcae8ef7b95c6ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731645601884 2024-11-15T04:40:03,386 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f6712c21246456d99c65ee336051d75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731645602547 2024-11-15T04:40:03,386 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/29f3890b42f7462b8acf34ab5088e123, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/07077e8db916429b8a31bfb679fce3e3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/941c215d519b48e7950bbca2e77a2956, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/b1b2d645438d440cb0dcb1cc0cc05965] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=53.4 K 2024-11-15T04:40:03,387 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29f3890b42f7462b8acf34ab5088e123, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731645601041 2024-11-15T04:40:03,388 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07077e8db916429b8a31bfb679fce3e3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731645601172 2024-11-15T04:40:03,389 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 941c215d519b48e7950bbca2e77a2956, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731645601884 2024-11-15T04:40:03,390 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1b2d645438d440cb0dcb1cc0cc05965, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731645602547 2024-11-15T04:40:03,408 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:03,409 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/2b5b26f2a66a48ea879d9aa855d1ea56 is 50, key is test_row_0/B:col10/1731645602547/Put/seqid=0 2024-11-15T04:40:03,413 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:03,414 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/75bdb84019554c329be4a1726cc88a5c is 50, key is test_row_0/A:col10/1731645602547/Put/seqid=0 2024-11-15T04:40:03,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741904_1080 (size=13051) 2024-11-15T04:40:03,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741903_1079 (size=13051) 2024-11-15T04:40:03,462 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/2b5b26f2a66a48ea879d9aa855d1ea56 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2b5b26f2a66a48ea879d9aa855d1ea56 2024-11-15T04:40:03,468 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/B of bffc5fba59ef21ffa1864f468203f7b0 into 2b5b26f2a66a48ea879d9aa855d1ea56(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:03,468 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:03,468 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/B, priority=12, startTime=1731645603383; duration=0sec 2024-11-15T04:40:03,468 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:03,468 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:B 2024-11-15T04:40:03,468 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:03,470 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:03,470 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/C is initiating minor compaction (all files) 2024-11-15T04:40:03,470 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/C in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:03,470 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c38726c09e5a42538a085d6e99769db3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/23f568fb1451467f82212898dca53761, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c7766476fa654372a720318f794539bf, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/26b492cd82914e778dba605983c6fe2e] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=48.7 K 2024-11-15T04:40:03,471 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c38726c09e5a42538a085d6e99769db3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731645601041 2024-11-15T04:40:03,471 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 23f568fb1451467f82212898dca53761, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731645601172 2024-11-15T04:40:03,471 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c7766476fa654372a720318f794539bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731645601884 2024-11-15T04:40:03,472 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 26b492cd82914e778dba605983c6fe2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731645602547 2024-11-15T04:40:03,483 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#C#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:03,483 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/d7a711294b4f447d92c7719a69eb26ac is 50, key is test_row_0/C:col10/1731645602547/Put/seqid=0 2024-11-15T04:40:03,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741905_1081 (size=13051) 2024-11-15T04:40:03,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:03,584 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:03,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T04:40:03,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/f17e1ad9662c48878e95cb6eac872e99 is 50, key is test_row_0/A:col10/1731645603574/Put/seqid=0 2024-11-15T04:40:03,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645663600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645663601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645663602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645663604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741906_1082 (size=14741) 2024-11-15T04:40:03,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645663605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645663712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645663713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645663713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645663713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645663723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,859 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/75bdb84019554c329be4a1726cc88a5c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/75bdb84019554c329be4a1726cc88a5c 2024-11-15T04:40:03,881 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/A of bffc5fba59ef21ffa1864f468203f7b0 into 75bdb84019554c329be4a1726cc88a5c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:03,881 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:03,881 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/A, priority=12, startTime=1731645603383; duration=0sec 2024-11-15T04:40:03,881 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:03,882 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:A 2024-11-15T04:40:03,899 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/d7a711294b4f447d92c7719a69eb26ac as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d7a711294b4f447d92c7719a69eb26ac 2024-11-15T04:40:03,906 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/C of bffc5fba59ef21ffa1864f468203f7b0 into d7a711294b4f447d92c7719a69eb26ac(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:03,906 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:03,906 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/C, priority=12, startTime=1731645603383; duration=0sec 2024-11-15T04:40:03,906 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:03,906 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:C 2024-11-15T04:40:03,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645663923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645663923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645663923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645663923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645663933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T04:40:03,973 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:03,973 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T04:40:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T04:40:03,976 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:03,977 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:03,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:04,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/f17e1ad9662c48878e95cb6eac872e99 2024-11-15T04:40:04,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/f4339f59d454487297dd02d6cc01693d is 50, key is test_row_0/B:col10/1731645603574/Put/seqid=0 2024-11-15T04:40:04,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741907_1083 (size=12301) 2024-11-15T04:40:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T04:40:04,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T04:40:04,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:04,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:04,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:04,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:04,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:04,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645664233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645664233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645664233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645664234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645664243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T04:40:04,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:04,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:04,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:04,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:04,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T04:40:04,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T04:40:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:04,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:04,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/f4339f59d454487297dd02d6cc01693d 2024-11-15T04:40:04,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/43d0a48f221844d2aecb85a45c063323 is 50, key is test_row_0/C:col10/1731645603574/Put/seqid=0 2024-11-15T04:40:04,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741908_1084 (size=12301) 2024-11-15T04:40:04,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/43d0a48f221844d2aecb85a45c063323 2024-11-15T04:40:04,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/f17e1ad9662c48878e95cb6eac872e99 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/f17e1ad9662c48878e95cb6eac872e99 2024-11-15T04:40:04,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/f17e1ad9662c48878e95cb6eac872e99, entries=200, sequenceid=358, filesize=14.4 K 2024-11-15T04:40:04,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/f4339f59d454487297dd02d6cc01693d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f4339f59d454487297dd02d6cc01693d 2024-11-15T04:40:04,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f4339f59d454487297dd02d6cc01693d, entries=150, sequenceid=358, filesize=12.0 K 2024-11-15T04:40:04,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/43d0a48f221844d2aecb85a45c063323 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/43d0a48f221844d2aecb85a45c063323 2024-11-15T04:40:04,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/43d0a48f221844d2aecb85a45c063323, entries=150, sequenceid=358, filesize=12.0 K 2024-11-15T04:40:04,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for bffc5fba59ef21ffa1864f468203f7b0 in 986ms, sequenceid=358, compaction requested=false 2024-11-15T04:40:04,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:04,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T04:40:04,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:04,596 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T04:40:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T04:40:04,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/13bfdee1e22c48b8a635ba6d2f45dc54 is 50, key is test_row_0/A:col10/1731645603602/Put/seqid=0 2024-11-15T04:40:04,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741909_1085 (size=12301) 2024-11-15T04:40:04,649 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/13bfdee1e22c48b8a635ba6d2f45dc54 2024-11-15T04:40:04,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/eda9a0e342e94de996c1ef81069304b5 is 50, key is test_row_0/B:col10/1731645603602/Put/seqid=0 2024-11-15T04:40:04,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741910_1086 (size=12301) 2024-11-15T04:40:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:04,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:04,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645664774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645664775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645664777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645664777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645664779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645664882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645664882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645664882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645664893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:04,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645664893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,078 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/eda9a0e342e94de996c1ef81069304b5 2024-11-15T04:40:05,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645665093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645665093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/a3410f1fae664fafa00f6e0773079551 is 50, key is test_row_0/C:col10/1731645603602/Put/seqid=0 2024-11-15T04:40:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645665096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645665103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645665103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T04:40:05,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741911_1087 (size=12301) 2024-11-15T04:40:05,127 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/a3410f1fae664fafa00f6e0773079551 2024-11-15T04:40:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/13bfdee1e22c48b8a635ba6d2f45dc54 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/13bfdee1e22c48b8a635ba6d2f45dc54 2024-11-15T04:40:05,146 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/13bfdee1e22c48b8a635ba6d2f45dc54, entries=150, sequenceid=372, filesize=12.0 K 2024-11-15T04:40:05,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/eda9a0e342e94de996c1ef81069304b5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/eda9a0e342e94de996c1ef81069304b5 2024-11-15T04:40:05,155 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/eda9a0e342e94de996c1ef81069304b5, entries=150, sequenceid=372, filesize=12.0 K 2024-11-15T04:40:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/a3410f1fae664fafa00f6e0773079551 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/a3410f1fae664fafa00f6e0773079551 2024-11-15T04:40:05,167 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/a3410f1fae664fafa00f6e0773079551, entries=150, sequenceid=372, filesize=12.0 K 2024-11-15T04:40:05,168 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for bffc5fba59ef21ffa1864f468203f7b0 in 572ms, sequenceid=372, compaction requested=true 2024-11-15T04:40:05,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:05,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:05,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T04:40:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T04:40:05,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T04:40:05,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1960 sec 2024-11-15T04:40:05,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.2050 sec 2024-11-15T04:40:05,406 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:05,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:40:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:05,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/c329013f508b4fb28e3440e080efc6b8 is 50, key is test_row_0/A:col10/1731645605403/Put/seqid=0 2024-11-15T04:40:05,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645665421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645665423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645665425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645665425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645665427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741912_1088 (size=12301) 2024-11-15T04:40:05,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/c329013f508b4fb28e3440e080efc6b8 2024-11-15T04:40:05,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/1a374e1df839492795101ac2478ac875 is 50, key is test_row_0/B:col10/1731645605403/Put/seqid=0 2024-11-15T04:40:05,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741913_1089 (size=12301) 2024-11-15T04:40:05,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/1a374e1df839492795101ac2478ac875 2024-11-15T04:40:05,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/0e71bbffec9c4fbba79f9e9f1600b7fe is 50, key is test_row_0/C:col10/1731645605403/Put/seqid=0 2024-11-15T04:40:05,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741914_1090 (size=12301) 2024-11-15T04:40:05,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/0e71bbffec9c4fbba79f9e9f1600b7fe 2024-11-15T04:40:05,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645665533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645665533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645665533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645665533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645665533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/c329013f508b4fb28e3440e080efc6b8 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c329013f508b4fb28e3440e080efc6b8 2024-11-15T04:40:05,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c329013f508b4fb28e3440e080efc6b8, entries=150, sequenceid=399, filesize=12.0 K 2024-11-15T04:40:05,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/1a374e1df839492795101ac2478ac875 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/1a374e1df839492795101ac2478ac875 2024-11-15T04:40:05,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/1a374e1df839492795101ac2478ac875, entries=150, sequenceid=399, filesize=12.0 K 2024-11-15T04:40:05,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/0e71bbffec9c4fbba79f9e9f1600b7fe as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/0e71bbffec9c4fbba79f9e9f1600b7fe 2024-11-15T04:40:05,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/0e71bbffec9c4fbba79f9e9f1600b7fe, entries=150, sequenceid=399, filesize=12.0 K 2024-11-15T04:40:05,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for bffc5fba59ef21ffa1864f468203f7b0 in 177ms, sequenceid=399, compaction requested=true 2024-11-15T04:40:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:05,584 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:05,584 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:05,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:05,586 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:05,586 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:05,586 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/A is initiating minor compaction (all files) 2024-11-15T04:40:05,586 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/B is initiating minor compaction (all files) 2024-11-15T04:40:05,586 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/B in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:05,586 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/A in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:05,586 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2b5b26f2a66a48ea879d9aa855d1ea56, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f4339f59d454487297dd02d6cc01693d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/eda9a0e342e94de996c1ef81069304b5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/1a374e1df839492795101ac2478ac875] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=48.8 K 2024-11-15T04:40:05,586 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/75bdb84019554c329be4a1726cc88a5c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/f17e1ad9662c48878e95cb6eac872e99, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/13bfdee1e22c48b8a635ba6d2f45dc54, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c329013f508b4fb28e3440e080efc6b8] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=51.2 K 2024-11-15T04:40:05,587 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75bdb84019554c329be4a1726cc88a5c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731645602547 2024-11-15T04:40:05,587 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b5b26f2a66a48ea879d9aa855d1ea56, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731645602547 2024-11-15T04:40:05,587 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting f17e1ad9662c48878e95cb6eac872e99, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731645603246 2024-11-15T04:40:05,588 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting f4339f59d454487297dd02d6cc01693d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731645603246 2024-11-15T04:40:05,588 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13bfdee1e22c48b8a635ba6d2f45dc54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731645603597 2024-11-15T04:40:05,588 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting eda9a0e342e94de996c1ef81069304b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731645603597 2024-11-15T04:40:05,589 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a374e1df839492795101ac2478ac875, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731645605403 2024-11-15T04:40:05,589 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting c329013f508b4fb28e3440e080efc6b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731645605403 2024-11-15T04:40:05,613 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:05,614 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/54d5196569ca4c37b37f886ca56d15e6 is 50, key is test_row_0/A:col10/1731645605403/Put/seqid=0 2024-11-15T04:40:05,630 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:05,630 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/0d3a8eefbc5748d4b2d2ffc0bda6ddad is 50, key is test_row_0/B:col10/1731645605403/Put/seqid=0 2024-11-15T04:40:05,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741915_1091 (size=13187) 2024-11-15T04:40:05,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741916_1092 (size=13187) 2024-11-15T04:40:05,693 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/0d3a8eefbc5748d4b2d2ffc0bda6ddad as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0d3a8eefbc5748d4b2d2ffc0bda6ddad 2024-11-15T04:40:05,716 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/B of bffc5fba59ef21ffa1864f468203f7b0 into 0d3a8eefbc5748d4b2d2ffc0bda6ddad(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:05,716 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:05,716 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/B, priority=12, startTime=1731645605584; duration=0sec 2024-11-15T04:40:05,716 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:05,716 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:B 2024-11-15T04:40:05,716 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:05,720 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:05,720 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/C is initiating minor compaction (all files) 2024-11-15T04:40:05,720 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/C in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:05,720 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d7a711294b4f447d92c7719a69eb26ac, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/43d0a48f221844d2aecb85a45c063323, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/a3410f1fae664fafa00f6e0773079551, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/0e71bbffec9c4fbba79f9e9f1600b7fe] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=48.8 K 2024-11-15T04:40:05,721 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting d7a711294b4f447d92c7719a69eb26ac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731645602547 2024-11-15T04:40:05,721 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 43d0a48f221844d2aecb85a45c063323, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731645603246 2024-11-15T04:40:05,722 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting a3410f1fae664fafa00f6e0773079551, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731645603597 2024-11-15T04:40:05,723 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e71bbffec9c4fbba79f9e9f1600b7fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731645605403 2024-11-15T04:40:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:05,745 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:05,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T04:40:05,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/91f673f07d254e909ddf71cbe6f67c32 is 50, key is test_row_0/A:col10/1731645605744/Put/seqid=0 2024-11-15T04:40:05,771 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#C#compaction#81 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:05,772 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/8a6d8da84eca41ecbfe2d16f821837d1 is 50, key is test_row_0/C:col10/1731645605403/Put/seqid=0 2024-11-15T04:40:05,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645665774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645665775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645665782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645665782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645665784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741917_1093 (size=13187) 2024-11-15T04:40:05,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741918_1094 (size=14741) 2024-11-15T04:40:05,812 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/8a6d8da84eca41ecbfe2d16f821837d1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8a6d8da84eca41ecbfe2d16f821837d1 2024-11-15T04:40:05,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/91f673f07d254e909ddf71cbe6f67c32 2024-11-15T04:40:05,826 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/C of bffc5fba59ef21ffa1864f468203f7b0 into 8a6d8da84eca41ecbfe2d16f821837d1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:05,826 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:05,826 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/C, priority=12, startTime=1731645605584; duration=0sec 2024-11-15T04:40:05,826 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:05,826 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:C 2024-11-15T04:40:05,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/ca1b8e9471134388872264dc230caee8 is 50, key is test_row_0/B:col10/1731645605744/Put/seqid=0 2024-11-15T04:40:05,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741919_1095 (size=12301) 2024-11-15T04:40:05,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/ca1b8e9471134388872264dc230caee8 2024-11-15T04:40:05,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/546d8cf41a7d4813861ae1f5edb06144 is 50, key is test_row_0/C:col10/1731645605744/Put/seqid=0 2024-11-15T04:40:05,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741920_1096 (size=12301) 2024-11-15T04:40:05,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645665893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645665893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645665893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645665893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645665894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,059 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/54d5196569ca4c37b37f886ca56d15e6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/54d5196569ca4c37b37f886ca56d15e6 2024-11-15T04:40:06,068 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/A of bffc5fba59ef21ffa1864f468203f7b0 into 54d5196569ca4c37b37f886ca56d15e6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:06,068 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:06,068 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/A, priority=12, startTime=1731645605584; duration=0sec 2024-11-15T04:40:06,068 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:06,068 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:A 2024-11-15T04:40:06,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645666103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645666103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645666103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645666103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645666112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T04:40:06,123 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:06,123 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T04:40:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T04:40:06,125 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:06,126 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:06,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T04:40:06,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T04:40:06,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:06,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:06,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:06,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/546d8cf41a7d4813861ae1f5edb06144 2024-11-15T04:40:06,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/91f673f07d254e909ddf71cbe6f67c32 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/91f673f07d254e909ddf71cbe6f67c32 2024-11-15T04:40:06,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/91f673f07d254e909ddf71cbe6f67c32, entries=200, sequenceid=412, filesize=14.4 K 2024-11-15T04:40:06,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/ca1b8e9471134388872264dc230caee8 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ca1b8e9471134388872264dc230caee8 2024-11-15T04:40:06,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ca1b8e9471134388872264dc230caee8, entries=150, sequenceid=412, filesize=12.0 K 2024-11-15T04:40:06,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/546d8cf41a7d4813861ae1f5edb06144 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/546d8cf41a7d4813861ae1f5edb06144 2024-11-15T04:40:06,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/546d8cf41a7d4813861ae1f5edb06144, entries=150, sequenceid=412, filesize=12.0 K 2024-11-15T04:40:06,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for bffc5fba59ef21ffa1864f468203f7b0 in 575ms, sequenceid=412, compaction requested=false 2024-11-15T04:40:06,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:06,415 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:06,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:40:06,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/d8a557605f064b07b469053fb794689f is 50, key is test_row_0/A:col10/1731645606414/Put/seqid=0 2024-11-15T04:40:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645666420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645666422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645666424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741921_1097 (size=17181) 2024-11-15T04:40:06,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645666425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/d8a557605f064b07b469053fb794689f 2024-11-15T04:40:06,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645666426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T04:40:06,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:06,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:06,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:06,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/e91bccd7d68348c98410838bd7854cc6 is 50, key is test_row_0/B:col10/1731645606414/Put/seqid=0 2024-11-15T04:40:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T04:40:06,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741922_1098 (size=12301) 2024-11-15T04:40:06,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645666533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645666533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645666533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645666533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T04:40:06,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:06,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:06,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:06,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T04:40:06,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:06,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:06,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:06,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645666743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645666743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645666743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645666743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T04:40:06,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/e91bccd7d68348c98410838bd7854cc6 2024-11-15T04:40:06,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/42faf69898a94f208db8a8f6ba8fb21c is 50, key is test_row_0/C:col10/1731645606414/Put/seqid=0 2024-11-15T04:40:06,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741923_1099 (size=12301) 2024-11-15T04:40:06,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T04:40:06,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:06,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:06,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:06,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:06,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645666932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T04:40:07,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:07,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:07,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:07,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:07,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:07,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:07,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645667052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645667062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645667063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645667063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T04:40:07,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:07,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:07,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:07,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:07,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:07,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T04:40:07,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/42faf69898a94f208db8a8f6ba8fb21c 2024-11-15T04:40:07,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/d8a557605f064b07b469053fb794689f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/d8a557605f064b07b469053fb794689f 2024-11-15T04:40:07,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/d8a557605f064b07b469053fb794689f, entries=250, sequenceid=442, filesize=16.8 K 2024-11-15T04:40:07,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/e91bccd7d68348c98410838bd7854cc6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/e91bccd7d68348c98410838bd7854cc6 2024-11-15T04:40:07,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/e91bccd7d68348c98410838bd7854cc6, entries=150, sequenceid=442, filesize=12.0 K 2024-11-15T04:40:07,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/42faf69898a94f208db8a8f6ba8fb21c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/42faf69898a94f208db8a8f6ba8fb21c 2024-11-15T04:40:07,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/42faf69898a94f208db8a8f6ba8fb21c, entries=150, sequenceid=442, filesize=12.0 K 2024-11-15T04:40:07,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for bffc5fba59ef21ffa1864f468203f7b0 in 881ms, sequenceid=442, compaction requested=true 2024-11-15T04:40:07,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:07,297 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:07,297 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:07,298 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:07,298 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/A is initiating minor compaction (all files) 2024-11-15T04:40:07,298 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/A in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:07,298 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/54d5196569ca4c37b37f886ca56d15e6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/91f673f07d254e909ddf71cbe6f67c32, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/d8a557605f064b07b469053fb794689f] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=44.1 K 2024-11-15T04:40:07,298 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:07,299 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/B is initiating minor compaction (all files) 2024-11-15T04:40:07,299 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54d5196569ca4c37b37f886ca56d15e6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731645605403 2024-11-15T04:40:07,299 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/B in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:07,299 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0d3a8eefbc5748d4b2d2ffc0bda6ddad, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ca1b8e9471134388872264dc230caee8, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/e91bccd7d68348c98410838bd7854cc6] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=36.9 K 2024-11-15T04:40:07,299 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91f673f07d254e909ddf71cbe6f67c32, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731645605420 2024-11-15T04:40:07,299 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d3a8eefbc5748d4b2d2ffc0bda6ddad, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731645605403 2024-11-15T04:40:07,299 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8a557605f064b07b469053fb794689f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731645605781 2024-11-15T04:40:07,300 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1b8e9471134388872264dc230caee8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731645605422 2024-11-15T04:40:07,300 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e91bccd7d68348c98410838bd7854cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731645605781 2024-11-15T04:40:07,311 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:07,311 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/8d00d61e8e2c4127866530b14ba873c9 is 50, key is test_row_0/B:col10/1731645606414/Put/seqid=0 2024-11-15T04:40:07,324 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:07,325 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/6e826a9925354ebbb0442e084edff0a6 is 50, key is test_row_0/A:col10/1731645606414/Put/seqid=0 2024-11-15T04:40:07,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741924_1100 (size=13289) 2024-11-15T04:40:07,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741925_1101 (size=13289) 2024-11-15T04:40:07,356 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/6e826a9925354ebbb0442e084edff0a6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6e826a9925354ebbb0442e084edff0a6 2024-11-15T04:40:07,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T04:40:07,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:07,359 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T04:40:07,364 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/A of bffc5fba59ef21ffa1864f468203f7b0 into 6e826a9925354ebbb0442e084edff0a6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:07,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/53f2c91c64214dcfaf84e57712bdcd2e is 50, key is test_row_0/A:col10/1731645606424/Put/seqid=0 2024-11-15T04:40:07,364 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:07,364 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/A, priority=13, startTime=1731645607297; duration=0sec 2024-11-15T04:40:07,364 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:07,364 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:A 2024-11-15T04:40:07,365 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:07,367 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:07,367 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/C is initiating minor compaction (all files) 2024-11-15T04:40:07,368 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/C in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:07,368 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8a6d8da84eca41ecbfe2d16f821837d1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/546d8cf41a7d4813861ae1f5edb06144, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/42faf69898a94f208db8a8f6ba8fb21c] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=36.9 K 2024-11-15T04:40:07,368 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a6d8da84eca41ecbfe2d16f821837d1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731645605403 2024-11-15T04:40:07,369 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 546d8cf41a7d4813861ae1f5edb06144, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731645605422 2024-11-15T04:40:07,369 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42faf69898a94f208db8a8f6ba8fb21c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731645605781 2024-11-15T04:40:07,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741926_1102 (size=12301) 2024-11-15T04:40:07,379 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#C#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:07,379 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/19d2029d1848472fb0335c82052b363c is 50, key is test_row_0/C:col10/1731645606414/Put/seqid=0 2024-11-15T04:40:07,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741927_1103 (size=13289) 2024-11-15T04:40:07,405 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/19d2029d1848472fb0335c82052b363c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/19d2029d1848472fb0335c82052b363c 2024-11-15T04:40:07,412 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/C of bffc5fba59ef21ffa1864f468203f7b0 into 19d2029d1848472fb0335c82052b363c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:07,412 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:07,412 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/C, priority=13, startTime=1731645607297; duration=0sec 2024-11-15T04:40:07,412 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:07,412 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:C 2024-11-15T04:40:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:07,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. as already flushing 2024-11-15T04:40:07,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645667607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645667608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645667609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645667610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645667712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645667722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645667723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645667723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,747 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/8d00d61e8e2c4127866530b14ba873c9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/8d00d61e8e2c4127866530b14ba873c9 2024-11-15T04:40:07,752 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/B of bffc5fba59ef21ffa1864f468203f7b0 into 8d00d61e8e2c4127866530b14ba873c9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:07,752 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:07,752 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/B, priority=13, startTime=1731645607297; duration=0sec 2024-11-15T04:40:07,752 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:07,752 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:B 2024-11-15T04:40:07,771 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/53f2c91c64214dcfaf84e57712bdcd2e 2024-11-15T04:40:07,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/63a8ab31d5384325be930733b71cde4d is 50, key is test_row_0/B:col10/1731645606424/Put/seqid=0 2024-11-15T04:40:07,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741928_1104 (size=12301) 2024-11-15T04:40:07,786 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/63a8ab31d5384325be930733b71cde4d 2024-11-15T04:40:07,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/4cd26c8ba32e448889c3e61652c5d2e7 is 50, key is test_row_0/C:col10/1731645606424/Put/seqid=0 2024-11-15T04:40:07,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741929_1105 (size=12301) 2024-11-15T04:40:07,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54308 deadline: 1731645667922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54288 deadline: 1731645667932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54342 deadline: 1731645667933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54320 deadline: 1731645667933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54330 deadline: 1731645667943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:07,949 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in bffc5fba59ef21ffa1864f468203f7b0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:07,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:07,996 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T04:40:07,996 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T04:40:07,996 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:07,996 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:08,000 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T04:40:08,000 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:08,001 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,001 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,001 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,001 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,002 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:08,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:08,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:08,002 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,007 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T04:40:08,007 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:08,007 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,007 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:08,199 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/4cd26c8ba32e448889c3e61652c5d2e7 2024-11-15T04:40:08,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/53f2c91c64214dcfaf84e57712bdcd2e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/53f2c91c64214dcfaf84e57712bdcd2e 2024-11-15T04:40:08,213 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/53f2c91c64214dcfaf84e57712bdcd2e, entries=150, sequenceid=451, filesize=12.0 K 2024-11-15T04:40:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/63a8ab31d5384325be930733b71cde4d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/63a8ab31d5384325be930733b71cde4d 2024-11-15T04:40:08,218 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/63a8ab31d5384325be930733b71cde4d, entries=150, sequenceid=451, filesize=12.0 K 2024-11-15T04:40:08,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/4cd26c8ba32e448889c3e61652c5d2e7 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4cd26c8ba32e448889c3e61652c5d2e7 2024-11-15T04:40:08,225 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4cd26c8ba32e448889c3e61652c5d2e7, entries=150, sequenceid=451, filesize=12.0 K 2024-11-15T04:40:08,226 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for bffc5fba59ef21ffa1864f468203f7b0 in 867ms, sequenceid=451, compaction requested=false 2024-11-15T04:40:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:08,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T04:40:08,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T04:40:08,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T04:40:08,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1020 sec 2024-11-15T04:40:08,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.1070 sec 2024-11-15T04:40:08,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:08,234 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:08,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-15T04:40:08,234 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T04:40:08,234 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:08,234 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,234 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:08,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/428e6383cc5041ddae5341b023714e9e is 50, key is test_row_0/A:col10/1731645607606/Put/seqid=0 2024-11-15T04:40:08,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741930_1106 (size=12301) 2024-11-15T04:40:08,245 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T04:40:08,245 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T04:40:08,245 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T04:40:08,245 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:08,245 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:08,245 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:08,245 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,245 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,245 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,245 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,245 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,245 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:08,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:08,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:08,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T04:40:08,273 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:08,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/428e6383cc5041ddae5341b023714e9e 2024-11-15T04:40:08,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/7d40274e07504b0f838e5699c3d3a2c4 is 50, key is test_row_0/B:col10/1731645607606/Put/seqid=0 2024-11-15T04:40:08,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741931_1107 (size=12301) 2024-11-15T04:40:09,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/7d40274e07504b0f838e5699c3d3a2c4 2024-11-15T04:40:09,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/90c1ddb5282f456ea7a2b91c15c6bf1c is 50, key is test_row_0/C:col10/1731645607606/Put/seqid=0 2024-11-15T04:40:09,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741932_1108 (size=12301) 2024-11-15T04:40:09,360 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T04:40:09,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/90c1ddb5282f456ea7a2b91c15c6bf1c 2024-11-15T04:40:09,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/428e6383cc5041ddae5341b023714e9e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/428e6383cc5041ddae5341b023714e9e 2024-11-15T04:40:09,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/428e6383cc5041ddae5341b023714e9e, entries=150, sequenceid=483, filesize=12.0 K 2024-11-15T04:40:09,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/7d40274e07504b0f838e5699c3d3a2c4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7d40274e07504b0f838e5699c3d3a2c4 2024-11-15T04:40:09,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7d40274e07504b0f838e5699c3d3a2c4, entries=150, sequenceid=483, filesize=12.0 K 2024-11-15T04:40:09,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/90c1ddb5282f456ea7a2b91c15c6bf1c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/90c1ddb5282f456ea7a2b91c15c6bf1c 2024-11-15T04:40:09,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/90c1ddb5282f456ea7a2b91c15c6bf1c, entries=150, sequenceid=483, filesize=12.0 K 2024-11-15T04:40:09,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=20.13 KB/20610 for bffc5fba59ef21ffa1864f468203f7b0 in 1282ms, sequenceid=483, compaction requested=true 2024-11-15T04:40:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:09,516 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:09,516 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bffc5fba59ef21ffa1864f468203f7b0:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:09,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:09,517 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:09,517 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:09,518 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/B is initiating minor compaction (all files) 2024-11-15T04:40:09,518 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/A is initiating minor compaction (all files) 2024-11-15T04:40:09,518 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/B in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:09,518 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/A in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:09,518 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/8d00d61e8e2c4127866530b14ba873c9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/63a8ab31d5384325be930733b71cde4d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7d40274e07504b0f838e5699c3d3a2c4] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=37.0 K 2024-11-15T04:40:09,518 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6e826a9925354ebbb0442e084edff0a6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/53f2c91c64214dcfaf84e57712bdcd2e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/428e6383cc5041ddae5341b023714e9e] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=37.0 K 2024-11-15T04:40:09,518 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d00d61e8e2c4127866530b14ba873c9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731645605781 2024-11-15T04:40:09,518 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e826a9925354ebbb0442e084edff0a6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731645605781 2024-11-15T04:40:09,519 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53f2c91c64214dcfaf84e57712bdcd2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731645606420 2024-11-15T04:40:09,519 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 63a8ab31d5384325be930733b71cde4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731645606420 2024-11-15T04:40:09,519 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 428e6383cc5041ddae5341b023714e9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731645607606 2024-11-15T04:40:09,519 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d40274e07504b0f838e5699c3d3a2c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731645607606 2024-11-15T04:40:09,529 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#B#compaction#96 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:09,529 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/e9f4070e2e214472a2a308b2ea8d6dad is 50, key is test_row_0/B:col10/1731645607606/Put/seqid=0 2024-11-15T04:40:09,532 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#A#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:09,533 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/cb0da2ba439d4bdcacd3582d7b0ea311 is 50, key is test_row_0/A:col10/1731645607606/Put/seqid=0 2024-11-15T04:40:09,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741933_1109 (size=13391) 2024-11-15T04:40:09,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741934_1110 (size=13391) 2024-11-15T04:40:09,952 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/e9f4070e2e214472a2a308b2ea8d6dad as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/e9f4070e2e214472a2a308b2ea8d6dad 2024-11-15T04:40:09,954 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/cb0da2ba439d4bdcacd3582d7b0ea311 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/cb0da2ba439d4bdcacd3582d7b0ea311 2024-11-15T04:40:09,961 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/B of bffc5fba59ef21ffa1864f468203f7b0 into e9f4070e2e214472a2a308b2ea8d6dad(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:09,961 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:09,961 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/B, priority=13, startTime=1731645609516; duration=0sec 2024-11-15T04:40:09,961 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/A of bffc5fba59ef21ffa1864f468203f7b0 into cb0da2ba439d4bdcacd3582d7b0ea311(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:09,961 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:09,961 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/A, priority=13, startTime=1731645609516; duration=0sec 2024-11-15T04:40:09,961 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:09,961 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:B 2024-11-15T04:40:09,961 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:09,961 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:A 2024-11-15T04:40:09,961 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:09,963 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:09,963 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): bffc5fba59ef21ffa1864f468203f7b0/C is initiating minor compaction (all files) 2024-11-15T04:40:09,963 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bffc5fba59ef21ffa1864f468203f7b0/C in TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:09,963 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/19d2029d1848472fb0335c82052b363c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4cd26c8ba32e448889c3e61652c5d2e7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/90c1ddb5282f456ea7a2b91c15c6bf1c] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp, totalSize=37.0 K 2024-11-15T04:40:09,964 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 19d2029d1848472fb0335c82052b363c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731645605781 2024-11-15T04:40:09,964 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T04:40:09,964 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:09,964 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:09,965 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd26c8ba32e448889c3e61652c5d2e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731645606420 2024-11-15T04:40:09,965 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:09,965 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:09,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T04:40:09,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-15T04:40:09,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-15T04:40:09,965 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-15T04:40:09,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-15T04:40:09,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-15T04:40:09,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T04:40:09,966 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 90c1ddb5282f456ea7a2b91c15c6bf1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1731645607606 2024-11-15T04:40:09,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2845 2024-11-15T04:40:09,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2836 2024-11-15T04:40:09,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T04:40:09,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1344 2024-11-15T04:40:09,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4024 rows 2024-11-15T04:40:09,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1350 2024-11-15T04:40:09,966 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4049 rows 2024-11-15T04:40:09,966 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T04:40:09,966 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T04:40:09,966 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:09,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:09,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:09,972 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T04:40:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:09,977 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bffc5fba59ef21ffa1864f468203f7b0#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:09,978 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/12d713214c35479a89526d79c0ec728c is 50, key is test_row_0/C:col10/1731645607606/Put/seqid=0 2024-11-15T04:40:09,982 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645609981"}]},"ts":"1731645609981"} 2024-11-15T04:40:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T04:40:09,984 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T04:40:09,984 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T04:40:09,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T04:40:09,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741935_1111 (size=13391) 2024-11-15T04:40:09,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bffc5fba59ef21ffa1864f468203f7b0, UNASSIGN}] 2024-11-15T04:40:09,995 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bffc5fba59ef21ffa1864f468203f7b0, UNASSIGN 2024-11-15T04:40:09,997 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/12d713214c35479a89526d79c0ec728c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/12d713214c35479a89526d79c0ec728c 2024-11-15T04:40:09,999 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=bffc5fba59ef21ffa1864f468203f7b0, regionState=CLOSING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:10,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bffc5fba59ef21ffa1864f468203f7b0, UNASSIGN because future has completed 2024-11-15T04:40:10,010 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T04:40:10,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:40:10,011 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bffc5fba59ef21ffa1864f468203f7b0/C of bffc5fba59ef21ffa1864f468203f7b0 into 12d713214c35479a89526d79c0ec728c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:10,011 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bffc5fba59ef21ffa1864f468203f7b0: 2024-11-15T04:40:10,011 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., storeName=bffc5fba59ef21ffa1864f468203f7b0/C, priority=13, startTime=1731645609516; duration=0sec 2024-11-15T04:40:10,011 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:10,011 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bffc5fba59ef21ffa1864f468203f7b0:C 2024-11-15T04:40:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T04:40:10,168 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:10,168 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T04:40:10,169 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing bffc5fba59ef21ffa1864f468203f7b0, disabling compactions & flushes 2024-11-15T04:40:10,169 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:10,169 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:10,169 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. after waiting 0 ms 2024-11-15T04:40:10,169 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:10,170 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing bffc5fba59ef21ffa1864f468203f7b0 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-15T04:40:10,175 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/e9138f3d7e97416b9f9fd02cb7d87fa6 is 50, key is test_row_0/A:col10/1731645609963/Put/seqid=0 2024-11-15T04:40:10,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741936_1112 (size=9857) 2024-11-15T04:40:10,181 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/e9138f3d7e97416b9f9fd02cb7d87fa6 2024-11-15T04:40:10,189 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/effae1419ce34491b16821b571fa5b13 is 50, key is test_row_0/B:col10/1731645609963/Put/seqid=0 2024-11-15T04:40:10,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741937_1113 (size=9857) 2024-11-15T04:40:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T04:40:10,595 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/effae1419ce34491b16821b571fa5b13 2024-11-15T04:40:10,604 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/ef8b30ac52bf4e058d7c977a3ef313c1 is 50, key is test_row_0/C:col10/1731645609963/Put/seqid=0 2024-11-15T04:40:10,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741938_1114 (size=9857) 2024-11-15T04:40:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T04:40:11,010 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/ef8b30ac52bf4e058d7c977a3ef313c1 2024-11-15T04:40:11,016 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/A/e9138f3d7e97416b9f9fd02cb7d87fa6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/e9138f3d7e97416b9f9fd02cb7d87fa6 2024-11-15T04:40:11,022 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/e9138f3d7e97416b9f9fd02cb7d87fa6, entries=100, sequenceid=493, filesize=9.6 K 2024-11-15T04:40:11,023 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/B/effae1419ce34491b16821b571fa5b13 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/effae1419ce34491b16821b571fa5b13 2024-11-15T04:40:11,029 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/effae1419ce34491b16821b571fa5b13, entries=100, sequenceid=493, filesize=9.6 K 2024-11-15T04:40:11,030 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/.tmp/C/ef8b30ac52bf4e058d7c977a3ef313c1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/ef8b30ac52bf4e058d7c977a3ef313c1 2024-11-15T04:40:11,036 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/ef8b30ac52bf4e058d7c977a3ef313c1, entries=100, sequenceid=493, filesize=9.6 K 2024-11-15T04:40:11,037 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for bffc5fba59ef21ffa1864f468203f7b0 in 868ms, sequenceid=493, compaction requested=false 2024-11-15T04:40:11,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/68b108ea5d544798b0f0e3388ec42b0c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fde0a0dbf00e407594e6d7199e9d2795, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9ea27444b32b4a84a13004a8cfd7e795, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/ed4d264c193d4aa181b79109192ba905, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/50bd26e121f848f4a42b9aed77cf4814, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/cdb325e6e465437490dd7e0ae96da197, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fb5168d07fd54b5488e370c84ee8b240, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c28c1317512a4fa0912ce38bcfbd7f27, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9973262c382a4b33b06b26cdb84241d5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/5cd89e46f9e3499eaecb917757a6aab5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6fdead09714b42e484e4eeecbd85cec1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6f809a9f03c245b8bdd7fe41af7a6ac8, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/828780f458a1471f8d603905f1925ea1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/3d5dc439a5bc4f4bad3c218dade34bde, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6b37c46c720f4e3497cd838e73f40270, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/e3c7b526f636406cb71414ebcf70d151, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/44ea9e74d3314235b9d9f98627cd2fd5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/29f3890b42f7462b8acf34ab5088e123, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/0448b5f3ed994c11904ecbb053f8eea2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/07077e8db916429b8a31bfb679fce3e3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/941c215d519b48e7950bbca2e77a2956, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/b1b2d645438d440cb0dcb1cc0cc05965, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/75bdb84019554c329be4a1726cc88a5c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/f17e1ad9662c48878e95cb6eac872e99, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/13bfdee1e22c48b8a635ba6d2f45dc54, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/54d5196569ca4c37b37f886ca56d15e6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c329013f508b4fb28e3440e080efc6b8, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/91f673f07d254e909ddf71cbe6f67c32, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/d8a557605f064b07b469053fb794689f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6e826a9925354ebbb0442e084edff0a6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/53f2c91c64214dcfaf84e57712bdcd2e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/428e6383cc5041ddae5341b023714e9e] to archive 2024-11-15T04:40:11,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:40:11,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/68b108ea5d544798b0f0e3388ec42b0c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/68b108ea5d544798b0f0e3388ec42b0c 2024-11-15T04:40:11,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fde0a0dbf00e407594e6d7199e9d2795 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fde0a0dbf00e407594e6d7199e9d2795 2024-11-15T04:40:11,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9ea27444b32b4a84a13004a8cfd7e795 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9ea27444b32b4a84a13004a8cfd7e795 2024-11-15T04:40:11,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/ed4d264c193d4aa181b79109192ba905 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/ed4d264c193d4aa181b79109192ba905 2024-11-15T04:40:11,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/50bd26e121f848f4a42b9aed77cf4814 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/50bd26e121f848f4a42b9aed77cf4814 2024-11-15T04:40:11,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/cdb325e6e465437490dd7e0ae96da197 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/cdb325e6e465437490dd7e0ae96da197 2024-11-15T04:40:11,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fb5168d07fd54b5488e370c84ee8b240 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/fb5168d07fd54b5488e370c84ee8b240 2024-11-15T04:40:11,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c28c1317512a4fa0912ce38bcfbd7f27 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c28c1317512a4fa0912ce38bcfbd7f27 2024-11-15T04:40:11,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9973262c382a4b33b06b26cdb84241d5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/9973262c382a4b33b06b26cdb84241d5 2024-11-15T04:40:11,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/5cd89e46f9e3499eaecb917757a6aab5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/5cd89e46f9e3499eaecb917757a6aab5 2024-11-15T04:40:11,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6fdead09714b42e484e4eeecbd85cec1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6fdead09714b42e484e4eeecbd85cec1 2024-11-15T04:40:11,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6f809a9f03c245b8bdd7fe41af7a6ac8 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6f809a9f03c245b8bdd7fe41af7a6ac8 2024-11-15T04:40:11,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/828780f458a1471f8d603905f1925ea1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/828780f458a1471f8d603905f1925ea1 2024-11-15T04:40:11,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/3d5dc439a5bc4f4bad3c218dade34bde to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/3d5dc439a5bc4f4bad3c218dade34bde 2024-11-15T04:40:11,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6b37c46c720f4e3497cd838e73f40270 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6b37c46c720f4e3497cd838e73f40270 2024-11-15T04:40:11,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/e3c7b526f636406cb71414ebcf70d151 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/e3c7b526f636406cb71414ebcf70d151 2024-11-15T04:40:11,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/44ea9e74d3314235b9d9f98627cd2fd5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/44ea9e74d3314235b9d9f98627cd2fd5 2024-11-15T04:40:11,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/29f3890b42f7462b8acf34ab5088e123 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/29f3890b42f7462b8acf34ab5088e123 2024-11-15T04:40:11,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/0448b5f3ed994c11904ecbb053f8eea2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/0448b5f3ed994c11904ecbb053f8eea2 2024-11-15T04:40:11,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/07077e8db916429b8a31bfb679fce3e3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/07077e8db916429b8a31bfb679fce3e3 2024-11-15T04:40:11,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/941c215d519b48e7950bbca2e77a2956 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/941c215d519b48e7950bbca2e77a2956 2024-11-15T04:40:11,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/b1b2d645438d440cb0dcb1cc0cc05965 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/b1b2d645438d440cb0dcb1cc0cc05965 2024-11-15T04:40:11,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/75bdb84019554c329be4a1726cc88a5c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/75bdb84019554c329be4a1726cc88a5c 2024-11-15T04:40:11,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/f17e1ad9662c48878e95cb6eac872e99 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/f17e1ad9662c48878e95cb6eac872e99 2024-11-15T04:40:11,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/13bfdee1e22c48b8a635ba6d2f45dc54 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/13bfdee1e22c48b8a635ba6d2f45dc54 2024-11-15T04:40:11,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/54d5196569ca4c37b37f886ca56d15e6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/54d5196569ca4c37b37f886ca56d15e6 2024-11-15T04:40:11,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c329013f508b4fb28e3440e080efc6b8 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/c329013f508b4fb28e3440e080efc6b8 2024-11-15T04:40:11,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/91f673f07d254e909ddf71cbe6f67c32 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/91f673f07d254e909ddf71cbe6f67c32 2024-11-15T04:40:11,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/d8a557605f064b07b469053fb794689f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/d8a557605f064b07b469053fb794689f 2024-11-15T04:40:11,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6e826a9925354ebbb0442e084edff0a6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/6e826a9925354ebbb0442e084edff0a6 2024-11-15T04:40:11,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/53f2c91c64214dcfaf84e57712bdcd2e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/53f2c91c64214dcfaf84e57712bdcd2e 2024-11-15T04:40:11,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/428e6383cc5041ddae5341b023714e9e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/428e6383cc5041ddae5341b023714e9e 2024-11-15T04:40:11,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/81b305d66b26498696f95f50efeab58f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/968235c733c04d809a374bf985d5cb5f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/44aa9f1365964518a0325db644b30d4e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/50e1c5e4a6194459ace2e56258bb6f23, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0e0518cd48df47f7a5a7f7ed70149904, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/32cfffec15484fa3a8f56d6952c4c28e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/834b2833f26c4ee3b184a8ff069022b3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/9dcf0080afa8446d8d9621c6a8fe6191, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0b5384df360d4c929dbcd528114bceed, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/3a61d04f5c114ff5997b8c70e8a803cc, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/9539b51d2c3546d2abb5f5d6b735ccf9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/410e280d55834e809489acb3f48f8984, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/5edc23d0f04746bebc5c1cf8829b70ab, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/009e47bd6fcb4e0f80edbe000c0252e9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ea67e326075e41bfa84a4a4157ab83b3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7bb25ccc35414e4ba5cc123ac816d6b3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ac4af21b244c4a66b2283de92f2c53ce, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/b447e70751c745509c92cbe5657a913f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/17d0c85f2cc74a04bc237bb87ee4d65a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/a022db5b1cb845cebd9f68418b75a222, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f8634dbbc53042678dcae8ef7b95c6ec, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2b5b26f2a66a48ea879d9aa855d1ea56, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2f6712c21246456d99c65ee336051d75, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f4339f59d454487297dd02d6cc01693d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/eda9a0e342e94de996c1ef81069304b5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0d3a8eefbc5748d4b2d2ffc0bda6ddad, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/1a374e1df839492795101ac2478ac875, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ca1b8e9471134388872264dc230caee8, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/8d00d61e8e2c4127866530b14ba873c9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/e91bccd7d68348c98410838bd7854cc6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/63a8ab31d5384325be930733b71cde4d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7d40274e07504b0f838e5699c3d3a2c4] to archive 2024-11-15T04:40:11,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:40:11,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/81b305d66b26498696f95f50efeab58f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/81b305d66b26498696f95f50efeab58f 2024-11-15T04:40:11,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/968235c733c04d809a374bf985d5cb5f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/968235c733c04d809a374bf985d5cb5f 2024-11-15T04:40:11,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/44aa9f1365964518a0325db644b30d4e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/44aa9f1365964518a0325db644b30d4e 2024-11-15T04:40:11,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/50e1c5e4a6194459ace2e56258bb6f23 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/50e1c5e4a6194459ace2e56258bb6f23 2024-11-15T04:40:11,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0e0518cd48df47f7a5a7f7ed70149904 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0e0518cd48df47f7a5a7f7ed70149904 2024-11-15T04:40:11,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/32cfffec15484fa3a8f56d6952c4c28e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/32cfffec15484fa3a8f56d6952c4c28e 2024-11-15T04:40:11,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/834b2833f26c4ee3b184a8ff069022b3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/834b2833f26c4ee3b184a8ff069022b3 2024-11-15T04:40:11,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/9dcf0080afa8446d8d9621c6a8fe6191 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/9dcf0080afa8446d8d9621c6a8fe6191 2024-11-15T04:40:11,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0b5384df360d4c929dbcd528114bceed to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0b5384df360d4c929dbcd528114bceed 2024-11-15T04:40:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T04:40:11,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/3a61d04f5c114ff5997b8c70e8a803cc to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/3a61d04f5c114ff5997b8c70e8a803cc 2024-11-15T04:40:11,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/9539b51d2c3546d2abb5f5d6b735ccf9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/9539b51d2c3546d2abb5f5d6b735ccf9 2024-11-15T04:40:11,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/410e280d55834e809489acb3f48f8984 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/410e280d55834e809489acb3f48f8984 2024-11-15T04:40:11,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/5edc23d0f04746bebc5c1cf8829b70ab to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/5edc23d0f04746bebc5c1cf8829b70ab 2024-11-15T04:40:11,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/009e47bd6fcb4e0f80edbe000c0252e9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/009e47bd6fcb4e0f80edbe000c0252e9 2024-11-15T04:40:11,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ea67e326075e41bfa84a4a4157ab83b3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ea67e326075e41bfa84a4a4157ab83b3 2024-11-15T04:40:11,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7bb25ccc35414e4ba5cc123ac816d6b3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7bb25ccc35414e4ba5cc123ac816d6b3 2024-11-15T04:40:11,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ac4af21b244c4a66b2283de92f2c53ce to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ac4af21b244c4a66b2283de92f2c53ce 2024-11-15T04:40:11,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/b447e70751c745509c92cbe5657a913f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/b447e70751c745509c92cbe5657a913f 2024-11-15T04:40:11,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/17d0c85f2cc74a04bc237bb87ee4d65a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/17d0c85f2cc74a04bc237bb87ee4d65a 2024-11-15T04:40:11,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/a022db5b1cb845cebd9f68418b75a222 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/a022db5b1cb845cebd9f68418b75a222 2024-11-15T04:40:11,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f8634dbbc53042678dcae8ef7b95c6ec to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f8634dbbc53042678dcae8ef7b95c6ec 2024-11-15T04:40:11,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2b5b26f2a66a48ea879d9aa855d1ea56 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2b5b26f2a66a48ea879d9aa855d1ea56 2024-11-15T04:40:11,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2f6712c21246456d99c65ee336051d75 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/2f6712c21246456d99c65ee336051d75 2024-11-15T04:40:11,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f4339f59d454487297dd02d6cc01693d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/f4339f59d454487297dd02d6cc01693d 2024-11-15T04:40:11,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/eda9a0e342e94de996c1ef81069304b5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/eda9a0e342e94de996c1ef81069304b5 2024-11-15T04:40:11,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0d3a8eefbc5748d4b2d2ffc0bda6ddad to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/0d3a8eefbc5748d4b2d2ffc0bda6ddad 2024-11-15T04:40:11,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/1a374e1df839492795101ac2478ac875 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/1a374e1df839492795101ac2478ac875 2024-11-15T04:40:11,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ca1b8e9471134388872264dc230caee8 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/ca1b8e9471134388872264dc230caee8 2024-11-15T04:40:11,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/8d00d61e8e2c4127866530b14ba873c9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/8d00d61e8e2c4127866530b14ba873c9 2024-11-15T04:40:11,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/e91bccd7d68348c98410838bd7854cc6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/e91bccd7d68348c98410838bd7854cc6 2024-11-15T04:40:11,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/63a8ab31d5384325be930733b71cde4d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/63a8ab31d5384325be930733b71cde4d 2024-11-15T04:40:11,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7d40274e07504b0f838e5699c3d3a2c4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/7d40274e07504b0f838e5699c3d3a2c4 2024-11-15T04:40:11,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/16836675119a4dd38fd7d36b188993d0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8ae82e46226940d4b6df324400f53cc5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/99e9d5f8403b4ef495fc2b09890e638e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/24232a027c744cd0b7ba5157260cdfc7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/94ef9b881cb940078cbd79b04569346b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4052a0e9f19845fe96ce7e6a86ac9af7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/be51123b6842457abc8f1f2f9c2a8100, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c4b81d7b6e6a41199fdff0c3bac51080, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/31d794a8381d491da4ebd04e6355ba43, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/5fdbb2d7ca444082b144160180719e8c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/ba4c5a740e3648648e26f9d9b05ad1fa, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/eec0d4251cf444c1b86b7a5a030ff7fa, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4a955837215a482983f4915dfdb7b15c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/cf23eea2544a4226a168814ecca41546, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/f3f2962867a64270ac2b52e3946d359b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d137d407344f43e682c471d84551dad1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/30e3050e1c5d4a0abbeea502d9ab5b65, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c38726c09e5a42538a085d6e99769db3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/fbe1a4c3fcc64b68bd4dcef6b18b6b5c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/23f568fb1451467f82212898dca53761, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c7766476fa654372a720318f794539bf, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d7a711294b4f447d92c7719a69eb26ac, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/26b492cd82914e778dba605983c6fe2e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/43d0a48f221844d2aecb85a45c063323, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/a3410f1fae664fafa00f6e0773079551, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8a6d8da84eca41ecbfe2d16f821837d1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/0e71bbffec9c4fbba79f9e9f1600b7fe, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/546d8cf41a7d4813861ae1f5edb06144, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/19d2029d1848472fb0335c82052b363c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/42faf69898a94f208db8a8f6ba8fb21c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4cd26c8ba32e448889c3e61652c5d2e7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/90c1ddb5282f456ea7a2b91c15c6bf1c] to archive 2024-11-15T04:40:11,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:40:11,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/16836675119a4dd38fd7d36b188993d0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/16836675119a4dd38fd7d36b188993d0 2024-11-15T04:40:11,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8ae82e46226940d4b6df324400f53cc5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8ae82e46226940d4b6df324400f53cc5 2024-11-15T04:40:11,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/99e9d5f8403b4ef495fc2b09890e638e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/99e9d5f8403b4ef495fc2b09890e638e 2024-11-15T04:40:11,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/24232a027c744cd0b7ba5157260cdfc7 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/24232a027c744cd0b7ba5157260cdfc7 2024-11-15T04:40:11,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/94ef9b881cb940078cbd79b04569346b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/94ef9b881cb940078cbd79b04569346b 2024-11-15T04:40:11,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4052a0e9f19845fe96ce7e6a86ac9af7 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4052a0e9f19845fe96ce7e6a86ac9af7 2024-11-15T04:40:11,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/be51123b6842457abc8f1f2f9c2a8100 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/be51123b6842457abc8f1f2f9c2a8100 2024-11-15T04:40:11,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c4b81d7b6e6a41199fdff0c3bac51080 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c4b81d7b6e6a41199fdff0c3bac51080 2024-11-15T04:40:11,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/31d794a8381d491da4ebd04e6355ba43 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/31d794a8381d491da4ebd04e6355ba43 2024-11-15T04:40:11,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/5fdbb2d7ca444082b144160180719e8c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/5fdbb2d7ca444082b144160180719e8c 2024-11-15T04:40:11,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/ba4c5a740e3648648e26f9d9b05ad1fa to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/ba4c5a740e3648648e26f9d9b05ad1fa 2024-11-15T04:40:11,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/eec0d4251cf444c1b86b7a5a030ff7fa to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/eec0d4251cf444c1b86b7a5a030ff7fa 2024-11-15T04:40:11,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4a955837215a482983f4915dfdb7b15c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4a955837215a482983f4915dfdb7b15c 2024-11-15T04:40:11,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/cf23eea2544a4226a168814ecca41546 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/cf23eea2544a4226a168814ecca41546 2024-11-15T04:40:11,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/f3f2962867a64270ac2b52e3946d359b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/f3f2962867a64270ac2b52e3946d359b 2024-11-15T04:40:11,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d137d407344f43e682c471d84551dad1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d137d407344f43e682c471d84551dad1 2024-11-15T04:40:11,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/30e3050e1c5d4a0abbeea502d9ab5b65 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/30e3050e1c5d4a0abbeea502d9ab5b65 2024-11-15T04:40:11,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c38726c09e5a42538a085d6e99769db3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c38726c09e5a42538a085d6e99769db3 2024-11-15T04:40:11,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/fbe1a4c3fcc64b68bd4dcef6b18b6b5c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/fbe1a4c3fcc64b68bd4dcef6b18b6b5c 2024-11-15T04:40:11,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/23f568fb1451467f82212898dca53761 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/23f568fb1451467f82212898dca53761 2024-11-15T04:40:11,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c7766476fa654372a720318f794539bf to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/c7766476fa654372a720318f794539bf 2024-11-15T04:40:11,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d7a711294b4f447d92c7719a69eb26ac to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/d7a711294b4f447d92c7719a69eb26ac 2024-11-15T04:40:11,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/26b492cd82914e778dba605983c6fe2e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/26b492cd82914e778dba605983c6fe2e 2024-11-15T04:40:11,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/43d0a48f221844d2aecb85a45c063323 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/43d0a48f221844d2aecb85a45c063323 2024-11-15T04:40:11,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/a3410f1fae664fafa00f6e0773079551 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/a3410f1fae664fafa00f6e0773079551 2024-11-15T04:40:11,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8a6d8da84eca41ecbfe2d16f821837d1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/8a6d8da84eca41ecbfe2d16f821837d1 2024-11-15T04:40:11,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/0e71bbffec9c4fbba79f9e9f1600b7fe to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/0e71bbffec9c4fbba79f9e9f1600b7fe 2024-11-15T04:40:11,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/546d8cf41a7d4813861ae1f5edb06144 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/546d8cf41a7d4813861ae1f5edb06144 2024-11-15T04:40:11,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/19d2029d1848472fb0335c82052b363c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/19d2029d1848472fb0335c82052b363c 2024-11-15T04:40:11,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/42faf69898a94f208db8a8f6ba8fb21c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/42faf69898a94f208db8a8f6ba8fb21c 2024-11-15T04:40:11,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4cd26c8ba32e448889c3e61652c5d2e7 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/4cd26c8ba32e448889c3e61652c5d2e7 2024-11-15T04:40:11,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/90c1ddb5282f456ea7a2b91c15c6bf1c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/90c1ddb5282f456ea7a2b91c15c6bf1c 2024-11-15T04:40:11,207 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/recovered.edits/496.seqid, newMaxSeqId=496, maxSeqId=1 2024-11-15T04:40:11,208 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. 2024-11-15T04:40:11,208 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for bffc5fba59ef21ffa1864f468203f7b0: Waiting for close lock at 1731645610169Running coprocessor pre-close hooks at 1731645610169Disabling compacts and flushes for region at 1731645610169Disabling writes for close at 1731645610169Obtaining lock to block concurrent updates at 1731645610170 (+1 ms)Preparing flush snapshotting stores in bffc5fba59ef21ffa1864f468203f7b0 at 1731645610170Finished memstore snapshotting TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731645610170Flushing stores of TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0. at 1731645610171 (+1 ms)Flushing bffc5fba59ef21ffa1864f468203f7b0/A: creating writer at 1731645610171Flushing bffc5fba59ef21ffa1864f468203f7b0/A: appending metadata at 1731645610174 (+3 ms)Flushing bffc5fba59ef21ffa1864f468203f7b0/A: closing flushed file at 1731645610174Flushing bffc5fba59ef21ffa1864f468203f7b0/B: creating writer at 1731645610186 (+12 ms)Flushing bffc5fba59ef21ffa1864f468203f7b0/B: appending metadata at 1731645610189 (+3 ms)Flushing bffc5fba59ef21ffa1864f468203f7b0/B: closing flushed file at 1731645610189Flushing bffc5fba59ef21ffa1864f468203f7b0/C: creating writer at 1731645610600 (+411 ms)Flushing bffc5fba59ef21ffa1864f468203f7b0/C: appending metadata at 1731645610603 (+3 ms)Flushing bffc5fba59ef21ffa1864f468203f7b0/C: closing flushed file at 1731645610604 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d055223: reopening flushed file at 1731645611015 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76c0060f: reopening flushed file at 1731645611022 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dc18c1e: reopening flushed file at 1731645611029 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for bffc5fba59ef21ffa1864f468203f7b0 in 868ms, sequenceid=493, compaction requested=false at 1731645611037 (+8 ms)Writing region close event to WAL at 1731645611195 (+158 ms)Running coprocessor post-close hooks at 1731645611208 (+13 ms)Closed at 1731645611208 2024-11-15T04:40:11,212 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:11,212 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=bffc5fba59ef21ffa1864f468203f7b0, regionState=CLOSED 2024-11-15T04:40:11,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:40:11,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T04:40:11,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure bffc5fba59ef21ffa1864f468203f7b0, server=26747e6ada4f,42935,1731645581086 in 1.2060 sec 2024-11-15T04:40:11,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-15T04:40:11,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bffc5fba59ef21ffa1864f468203f7b0, UNASSIGN in 1.2250 sec 2024-11-15T04:40:11,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T04:40:11,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2350 sec 2024-11-15T04:40:11,225 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645611224"}]},"ts":"1731645611224"} 2024-11-15T04:40:11,227 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T04:40:11,227 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T04:40:11,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2530 sec 2024-11-15T04:40:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T04:40:12,133 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:12,136 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T04:40:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:12,141 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T04:40:12,143 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:12,148 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:12,153 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/recovered.edits] 2024-11-15T04:40:12,156 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/cb0da2ba439d4bdcacd3582d7b0ea311 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/cb0da2ba439d4bdcacd3582d7b0ea311 2024-11-15T04:40:12,157 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/e9138f3d7e97416b9f9fd02cb7d87fa6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/A/e9138f3d7e97416b9f9fd02cb7d87fa6 2024-11-15T04:40:12,159 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/e9f4070e2e214472a2a308b2ea8d6dad to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/e9f4070e2e214472a2a308b2ea8d6dad 2024-11-15T04:40:12,161 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/effae1419ce34491b16821b571fa5b13 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/B/effae1419ce34491b16821b571fa5b13 2024-11-15T04:40:12,163 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/12d713214c35479a89526d79c0ec728c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/12d713214c35479a89526d79c0ec728c 2024-11-15T04:40:12,164 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/ef8b30ac52bf4e058d7c977a3ef313c1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/C/ef8b30ac52bf4e058d7c977a3ef313c1 2024-11-15T04:40:12,167 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/recovered.edits/496.seqid to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0/recovered.edits/496.seqid 2024-11-15T04:40:12,167 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/bffc5fba59ef21ffa1864f468203f7b0 2024-11-15T04:40:12,167 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T04:40:12,172 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:12,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T04:40:12,183 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T04:40:12,210 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T04:40:12,212 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:12,212 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T04:40:12,212 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731645612212"}]},"ts":"9223372036854775807"} 2024-11-15T04:40:12,216 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T04:40:12,216 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bffc5fba59ef21ffa1864f468203f7b0, NAME => 'TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T04:40:12,216 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T04:40:12,216 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731645612216"}]},"ts":"9223372036854775807"} 2024-11-15T04:40:12,219 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T04:40:12,220 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:12,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-11-15T04:40:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T04:40:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T04:40:12,254 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:12,264 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=240 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1356923970_22 at /127.0.0.1:49592 [Waiting for operation #166] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1761888392_22 at /127.0.0.1:36428 [Waiting for operation #239] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;26747e6ada4f:42935-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1356923970_22 at /127.0.0.1:51604 [Waiting for operation #218] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1761888392_22 at /127.0.0.1:40840 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 238) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 12), AvailableMemoryMB=15264 (was 15407) 2024-11-15T04:40:12,272 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=15261 2024-11-15T04:40:12,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T04:40:12,274 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T04:40:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:12,276 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T04:40:12,276 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:12,276 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-15T04:40:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T04:40:12,277 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T04:40:12,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741939_1115 (size=959) 2024-11-15T04:40:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T04:40:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T04:40:12,685 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6 2024-11-15T04:40:12,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741940_1116 (size=53) 2024-11-15T04:40:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T04:40:13,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:40:13,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 465c1abe9e468bea9e6a5aa6fe32be31, disabling compactions & flushes 2024-11-15T04:40:13,091 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:13,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:13,092 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. after waiting 0 ms 2024-11-15T04:40:13,092 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:13,092 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:13,092 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 465c1abe9e468bea9e6a5aa6fe32be31: Waiting for close lock at 1731645613091Disabling compacts and flushes for region at 1731645613091Disabling writes for close at 1731645613092 (+1 ms)Writing region close event to WAL at 1731645613092Closed at 1731645613092 2024-11-15T04:40:13,093 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T04:40:13,094 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731645613093"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731645613093"}]},"ts":"1731645613093"} 2024-11-15T04:40:13,095 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T04:40:13,097 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T04:40:13,097 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645613097"}]},"ts":"1731645613097"} 2024-11-15T04:40:13,099 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T04:40:13,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, ASSIGN}] 2024-11-15T04:40:13,101 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, ASSIGN 2024-11-15T04:40:13,102 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, ASSIGN; state=OFFLINE, location=26747e6ada4f,42935,1731645581086; forceNewPlan=false, retain=false 2024-11-15T04:40:13,253 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=465c1abe9e468bea9e6a5aa6fe32be31, regionState=OPENING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:13,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, ASSIGN because future has completed 2024-11-15T04:40:13,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:40:13,412 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:13,412 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:40:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T04:40:13,413 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,413 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:40:13,413 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,413 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,415 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,419 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 465c1abe9e468bea9e6a5aa6fe32be31 columnFamilyName A 2024-11-15T04:40:13,419 DEBUG [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:13,419 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(327): Store=465c1abe9e468bea9e6a5aa6fe32be31/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:40:13,419 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,421 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 465c1abe9e468bea9e6a5aa6fe32be31 columnFamilyName B 2024-11-15T04:40:13,421 DEBUG [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:13,421 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(327): Store=465c1abe9e468bea9e6a5aa6fe32be31/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:40:13,421 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,422 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 465c1abe9e468bea9e6a5aa6fe32be31 columnFamilyName C 2024-11-15T04:40:13,422 DEBUG [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:13,423 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(327): Store=465c1abe9e468bea9e6a5aa6fe32be31/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:40:13,423 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,424 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,424 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,425 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,425 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,426 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:40:13,427 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,429 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T04:40:13,430 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 465c1abe9e468bea9e6a5aa6fe32be31; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74282756, jitterRate=0.10689932107925415}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:40:13,430 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:13,431 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 465c1abe9e468bea9e6a5aa6fe32be31: Running coprocessor pre-open hook at 1731645613413Writing region info on filesystem at 1731645613413Initializing all the Stores at 1731645613414 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645613415 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645613415Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645613415Cleaning up temporary data from old regions at 1731645613425 (+10 ms)Running coprocessor post-open hooks at 1731645613430 (+5 ms)Region opened successfully at 1731645613431 (+1 ms) 2024-11-15T04:40:13,432 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., pid=30, masterSystemTime=1731645613407 2024-11-15T04:40:13,433 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:13,433 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:13,434 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=465c1abe9e468bea9e6a5aa6fe32be31, regionState=OPEN, openSeqNum=2, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:13,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:40:13,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-15T04:40:13,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 in 182 msec 2024-11-15T04:40:13,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-15T04:40:13,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, ASSIGN in 340 msec 2024-11-15T04:40:13,443 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T04:40:13,443 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645613443"}]},"ts":"1731645613443"} 2024-11-15T04:40:13,444 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T04:40:13,445 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T04:40:13,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-11-15T04:40:14,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-15T04:40:14,423 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:14,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:14,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:14,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:14,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b44f32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:14,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:14,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:14,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81dc58f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:14,428 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:14,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:14,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:14,432 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T04:40:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 26747e6ada4f,42655,1731645580243 2024-11-15T04:40:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@51700bd0 2024-11-15T04:40:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T04:40:14,435 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T04:40:14,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T04:40:14,442 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T04:40:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:14,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741941_1117 (size=995) 2024-11-15T04:40:14,861 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T04:40:14,861 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T04:40:14,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T04:40:14,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, REOPEN/MOVE}] 2024-11-15T04:40:14,872 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, REOPEN/MOVE 2024-11-15T04:40:14,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=465c1abe9e468bea9e6a5aa6fe32be31, regionState=CLOSING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:14,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, REOPEN/MOVE because future has completed 2024-11-15T04:40:14,874 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T04:40:14,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:40:15,027 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,027 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T04:40:15,027 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 465c1abe9e468bea9e6a5aa6fe32be31, disabling compactions & flushes 2024-11-15T04:40:15,028 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,028 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,028 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. after waiting 0 ms 2024-11-15T04:40:15,028 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,032 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T04:40:15,035 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,035 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 465c1abe9e468bea9e6a5aa6fe32be31: Waiting for close lock at 1731645615027Running coprocessor pre-close hooks at 1731645615027Disabling compacts and flushes for region at 1731645615027Disabling writes for close at 1731645615028 (+1 ms)Writing region close event to WAL at 1731645615028Running coprocessor post-close hooks at 1731645615035 (+7 ms)Closed at 1731645615035 2024-11-15T04:40:15,035 WARN [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 465c1abe9e468bea9e6a5aa6fe32be31 to self. 2024-11-15T04:40:15,038 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,039 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=465c1abe9e468bea9e6a5aa6fe32be31, regionState=CLOSED 2024-11-15T04:40:15,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:40:15,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T04:40:15,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 in 168 msec 2024-11-15T04:40:15,045 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, REOPEN/MOVE; state=CLOSED, location=26747e6ada4f,42935,1731645581086; forceNewPlan=false, retain=true 2024-11-15T04:40:15,196 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=465c1abe9e468bea9e6a5aa6fe32be31, regionState=OPENING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, REOPEN/MOVE because future has completed 2024-11-15T04:40:15,198 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:40:15,355 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,355 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:40:15,356 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,356 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:40:15,356 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,356 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,363 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,371 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 465c1abe9e468bea9e6a5aa6fe32be31 columnFamilyName A 2024-11-15T04:40:15,374 DEBUG [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:15,375 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(327): Store=465c1abe9e468bea9e6a5aa6fe32be31/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:40:15,375 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,376 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 465c1abe9e468bea9e6a5aa6fe32be31 columnFamilyName B 2024-11-15T04:40:15,376 DEBUG [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:15,377 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(327): Store=465c1abe9e468bea9e6a5aa6fe32be31/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:40:15,377 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,377 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 465c1abe9e468bea9e6a5aa6fe32be31 columnFamilyName C 2024-11-15T04:40:15,377 DEBUG [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:15,378 INFO [StoreOpener-465c1abe9e468bea9e6a5aa6fe32be31-1 {}] regionserver.HStore(327): Store=465c1abe9e468bea9e6a5aa6fe32be31/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:40:15,378 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,378 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,379 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,380 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,380 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,381 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:40:15,382 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,383 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 465c1abe9e468bea9e6a5aa6fe32be31; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64084685, jitterRate=-0.04506377875804901}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:40:15,383 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,384 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 465c1abe9e468bea9e6a5aa6fe32be31: Running coprocessor pre-open hook at 1731645615356Writing region info on filesystem at 1731645615356Initializing all the Stores at 1731645615357 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645615357Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645615358 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645615358Cleaning up temporary data from old regions at 1731645615380 (+22 ms)Running coprocessor post-open hooks at 1731645615383 (+3 ms)Region opened successfully at 1731645615384 (+1 ms) 2024-11-15T04:40:15,385 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., pid=35, masterSystemTime=1731645615350 2024-11-15T04:40:15,388 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,388 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,389 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=465c1abe9e468bea9e6a5aa6fe32be31, regionState=OPEN, openSeqNum=5, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:40:15,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-15T04:40:15,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 in 194 msec 2024-11-15T04:40:15,397 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-15T04:40:15,397 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, REOPEN/MOVE in 524 msec 2024-11-15T04:40:15,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-15T04:40:15,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 534 msec 2024-11-15T04:40:15,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 958 msec 2024-11-15T04:40:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T04:40:15,407 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:15,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13460079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:15,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:15,410 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b81587e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:15,412 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:15,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64bbd0af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b1860b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:15,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:15,415 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7a241d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:15,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:15,418 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:15,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b70f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c7c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:15,419 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:15,421 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603141d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:15,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:15,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:15,426 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13147b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52903b3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:15,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:15,429 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5017cc17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:15,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:15,432 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f67acb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc5db83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:15,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:15,436 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9833d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:15,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:15,438 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:15,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a93159b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fbca19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,440 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:15,440 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:15,441 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a2be702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:15,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:15,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3114a47c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c562e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:15,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:15,446 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2559f6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:15,448 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:15,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6180c2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdf07c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:15,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:15,450 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f488d99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:15,452 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee88c70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59fb429c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:15,454 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:15,455 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ed6a82e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:15,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:15,456 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:15,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6623f6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:15,459 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:15,459 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:15,459 DEBUG [Thread-606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:15,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:15,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:15,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:40:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:40:15,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:40:15,475 DEBUG [Thread-608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:15,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-15T04:40:15,477 DEBUG [Thread-608 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:15,477 DEBUG [Thread-608 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T04:40:15,478 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:15,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:15,482 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:40:15,482 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:15,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:15,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:15,490 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:15,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:40:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:40:15,495 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:15,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:15,498 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:15,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:15,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:15,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:40:15,500 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:15,502 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:15,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:40:15,503 DEBUG [Thread-600 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:15,504 DEBUG [Thread-600 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:15,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:15,507 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:15,507 DEBUG [Thread-600 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:40:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:15,510 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:15,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:40:15,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152a5fa02f1ec844268fe6d3ad7bcc280b_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645615489/Put/seqid=0 2024-11-15T04:40:15,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741942_1118 (size=21918) 2024-11-15T04:40:15,579 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:15,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645675570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T04:40:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152a5fa02f1ec844268fe6d3ad7bcc280b_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152a5fa02f1ec844268fe6d3ad7bcc280b_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:15,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/b091d84c6cac497a941aba0a31016212, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:15,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/b091d84c6cac497a941aba0a31016212 is 175, key is test_row_0/A:col10/1731645615489/Put/seqid=0 2024-11-15T04:40:15,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645675578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645675578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645675581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645675582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741943_1119 (size=65323) 2024-11-15T04:40:15,617 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/b091d84c6cac497a941aba0a31016212 2024-11-15T04:40:15,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T04:40:15,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:15,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:15,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:15,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/c654b87037b6492c86dd4e79e13d75e7 is 50, key is test_row_0/B:col10/1731645615489/Put/seqid=0 2024-11-15T04:40:15,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645675693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741944_1120 (size=12001) 2024-11-15T04:40:15,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645675713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645675713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645675716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645675716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T04:40:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:15,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T04:40:15,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645675903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645675922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645675923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645675923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645675936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:15,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:15,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:15,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T04:40:15,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:15,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:15,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T04:40:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:16,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/c654b87037b6492c86dd4e79e13d75e7 2024-11-15T04:40:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T04:40:16,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/20afd1e7e2dd4908a369dc8734635702 is 50, key is test_row_0/C:col10/1731645615489/Put/seqid=0 2024-11-15T04:40:16,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741945_1121 (size=12001) 2024-11-15T04:40:16,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645676214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645676233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645676233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645676234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645676243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T04:40:16,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:16,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:16,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:16,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T04:40:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:16,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/20afd1e7e2dd4908a369dc8734635702 2024-11-15T04:40:16,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T04:40:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:16,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:16,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:16,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:16,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/b091d84c6cac497a941aba0a31016212 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b091d84c6cac497a941aba0a31016212 2024-11-15T04:40:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T04:40:16,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b091d84c6cac497a941aba0a31016212, entries=350, sequenceid=15, filesize=63.8 K 2024-11-15T04:40:16,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/c654b87037b6492c86dd4e79e13d75e7 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c654b87037b6492c86dd4e79e13d75e7 2024-11-15T04:40:16,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c654b87037b6492c86dd4e79e13d75e7, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T04:40:16,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/20afd1e7e2dd4908a369dc8734635702 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/20afd1e7e2dd4908a369dc8734635702 2024-11-15T04:40:16,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/20afd1e7e2dd4908a369dc8734635702, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T04:40:16,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1158ms, sequenceid=15, compaction requested=false 2024-11-15T04:40:16,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:16,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-15T04:40:16,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:16,722 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:40:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:16,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:16,759 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T04:40:16,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ec5088faf96f489897ca94ee760706e3_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645615566/Put/seqid=0 2024-11-15T04:40:16,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645676769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645676769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645676776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645676775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645676776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741946_1122 (size=12154) 2024-11-15T04:40:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:16,824 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ec5088faf96f489897ca94ee760706e3_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ec5088faf96f489897ca94ee760706e3_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/507d7fb728374570bd0f50ffc95f166a, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:16,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/507d7fb728374570bd0f50ffc95f166a is 175, key is test_row_0/A:col10/1731645615566/Put/seqid=0 2024-11-15T04:40:16,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741947_1123 (size=30955) 2024-11-15T04:40:16,849 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/507d7fb728374570bd0f50ffc95f166a 2024-11-15T04:40:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/e41dd12cbfc149a483289831d4a57dd9 is 50, key is test_row_0/B:col10/1731645615566/Put/seqid=0 2024-11-15T04:40:16,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741948_1124 (size=12001) 2024-11-15T04:40:16,890 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/e41dd12cbfc149a483289831d4a57dd9 2024-11-15T04:40:16,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645676893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645676894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645676894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645676894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645676894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:16,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/9689c61b42c94973964391a37edfb40c is 50, key is test_row_0/C:col10/1731645615566/Put/seqid=0 2024-11-15T04:40:16,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741949_1125 (size=12001) 2024-11-15T04:40:16,929 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/9689c61b42c94973964391a37edfb40c 2024-11-15T04:40:16,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/507d7fb728374570bd0f50ffc95f166a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/507d7fb728374570bd0f50ffc95f166a 2024-11-15T04:40:16,948 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/507d7fb728374570bd0f50ffc95f166a, entries=150, sequenceid=40, filesize=30.2 K 2024-11-15T04:40:16,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/e41dd12cbfc149a483289831d4a57dd9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e41dd12cbfc149a483289831d4a57dd9 2024-11-15T04:40:16,957 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e41dd12cbfc149a483289831d4a57dd9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T04:40:16,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/9689c61b42c94973964391a37edfb40c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9689c61b42c94973964391a37edfb40c 2024-11-15T04:40:16,972 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9689c61b42c94973964391a37edfb40c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T04:40:16,974 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 465c1abe9e468bea9e6a5aa6fe32be31 in 252ms, sequenceid=40, compaction requested=false 2024-11-15T04:40:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-15T04:40:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-15T04:40:16,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-15T04:40:16,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4940 sec 2024-11-15T04:40:16,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.5050 sec 2024-11-15T04:40:17,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:17,119 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:17,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T04:40:17,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155630a5d300bf4c91b42af0150ae0a4d4_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645616769/Put/seqid=0 2024-11-15T04:40:17,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645677171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645677169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645677181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645677183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645677185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741950_1126 (size=17034) 2024-11-15T04:40:17,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645677292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645677303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645677303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645677304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645677304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645677502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645677513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645677513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645677513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645677515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,614 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:17,619 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155630a5d300bf4c91b42af0150ae0a4d4_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155630a5d300bf4c91b42af0150ae0a4d4_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:17,620 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/1c2411b33f274a5b812bfd7586b6c41d, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:17,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/1c2411b33f274a5b812bfd7586b6c41d is 175, key is test_row_0/A:col10/1731645616769/Put/seqid=0 2024-11-15T04:40:17,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-15T04:40:17,624 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:17,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:17,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-15T04:40:17,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T04:40:17,627 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:17,627 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:17,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:17,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741951_1127 (size=48139) 2024-11-15T04:40:17,633 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/1c2411b33f274a5b812bfd7586b6c41d 2024-11-15T04:40:17,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/651107f6f0e84d2cb4b828b4836b5652 is 50, key is test_row_0/B:col10/1731645616769/Put/seqid=0 2024-11-15T04:40:17,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741952_1128 (size=12001) 2024-11-15T04:40:17,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/651107f6f0e84d2cb4b828b4836b5652 2024-11-15T04:40:17,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/8beb2789990c42cf8a8ad529b31e25c3 is 50, key is test_row_0/C:col10/1731645616769/Put/seqid=0 2024-11-15T04:40:17,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741953_1129 (size=12001) 2024-11-15T04:40:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T04:40:17,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T04:40:17,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:17,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:17,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:17,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:17,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:17,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645677812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645677823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645677823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645677823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645677833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:17,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T04:40:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:17,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T04:40:18,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T04:40:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:18,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:18,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/8beb2789990c42cf8a8ad529b31e25c3 2024-11-15T04:40:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/1c2411b33f274a5b812bfd7586b6c41d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/1c2411b33f274a5b812bfd7586b6c41d 2024-11-15T04:40:18,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/1c2411b33f274a5b812bfd7586b6c41d, entries=250, sequenceid=53, filesize=47.0 K 2024-11-15T04:40:18,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/651107f6f0e84d2cb4b828b4836b5652 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/651107f6f0e84d2cb4b828b4836b5652 2024-11-15T04:40:18,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/651107f6f0e84d2cb4b828b4836b5652, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T04:40:18,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/8beb2789990c42cf8a8ad529b31e25c3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8beb2789990c42cf8a8ad529b31e25c3 2024-11-15T04:40:18,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8beb2789990c42cf8a8ad529b31e25c3, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T04:40:18,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1041ms, sequenceid=53, compaction requested=true 2024-11-15T04:40:18,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:18,161 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:18,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:18,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:18,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:18,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:18,161 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:18,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:18,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:18,162 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:18,162 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/A is initiating minor compaction (all files) 2024-11-15T04:40:18,162 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/A in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:18,162 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b091d84c6cac497a941aba0a31016212, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/507d7fb728374570bd0f50ffc95f166a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/1c2411b33f274a5b812bfd7586b6c41d] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=141.0 K 2024-11-15T04:40:18,163 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:18,163 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b091d84c6cac497a941aba0a31016212, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/507d7fb728374570bd0f50ffc95f166a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/1c2411b33f274a5b812bfd7586b6c41d] 2024-11-15T04:40:18,163 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:18,163 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/B is initiating minor compaction (all files) 2024-11-15T04:40:18,163 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b091d84c6cac497a941aba0a31016212, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731645615473 2024-11-15T04:40:18,163 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/B in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:18,164 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c654b87037b6492c86dd4e79e13d75e7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e41dd12cbfc149a483289831d4a57dd9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/651107f6f0e84d2cb4b828b4836b5652] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=35.2 K 2024-11-15T04:40:18,164 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 507d7fb728374570bd0f50ffc95f166a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731645615566 2024-11-15T04:40:18,164 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c2411b33f274a5b812bfd7586b6c41d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731645616761 2024-11-15T04:40:18,164 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c654b87037b6492c86dd4e79e13d75e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731645615475 2024-11-15T04:40:18,165 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e41dd12cbfc149a483289831d4a57dd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731645615566 2024-11-15T04:40:18,165 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 651107f6f0e84d2cb4b828b4836b5652, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731645616769 2024-11-15T04:40:18,188 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:18,188 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/1beba43a83d740d29f51cdd290b56b42 is 50, key is test_row_0/B:col10/1731645616769/Put/seqid=0 2024-11-15T04:40:18,205 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:18,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741954_1130 (size=12104) 2024-11-15T04:40:18,231 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115947cede129fa4bbbaef4c75fb2a9aa95_465c1abe9e468bea9e6a5aa6fe32be31 store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:18,237 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115947cede129fa4bbbaef4c75fb2a9aa95_465c1abe9e468bea9e6a5aa6fe32be31, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:18,237 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115947cede129fa4bbbaef4c75fb2a9aa95_465c1abe9e468bea9e6a5aa6fe32be31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:18,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-15T04:40:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:18,245 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T04:40:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T04:40:18,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741955_1131 (size=4469) 2024-11-15T04:40:18,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115276e12b21290490c9dfe599ffc946366_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645617166/Put/seqid=0 2024-11-15T04:40:18,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741956_1132 (size=12154) 2024-11-15T04:40:18,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:18,326 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115276e12b21290490c9dfe599ffc946366_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115276e12b21290490c9dfe599ffc946366_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/9bafd5140adb4ad58ea5bcafdc22700a, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/9bafd5140adb4ad58ea5bcafdc22700a is 175, key is test_row_0/A:col10/1731645617166/Put/seqid=0 2024-11-15T04:40:18,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:18,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:18,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645678350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645678355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645678357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645678361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645678363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741957_1133 (size=30955) 2024-11-15T04:40:18,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645678473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645678473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645678474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645678483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645678483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,638 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/1beba43a83d740d29f51cdd290b56b42 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/1beba43a83d740d29f51cdd290b56b42 2024-11-15T04:40:18,658 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/B of 465c1abe9e468bea9e6a5aa6fe32be31 into 1beba43a83d740d29f51cdd290b56b42(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:18,658 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:18,658 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/B, priority=13, startTime=1731645618161; duration=0sec 2024-11-15T04:40:18,658 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:18,658 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:B 2024-11-15T04:40:18,658 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:18,663 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#A#compaction#112 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:18,663 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/26ca97a4adc84a9e8be873380a6c7851 is 175, key is test_row_0/A:col10/1731645616769/Put/seqid=0 2024-11-15T04:40:18,665 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:18,665 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/C is initiating minor compaction (all files) 2024-11-15T04:40:18,665 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/C in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:18,665 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/20afd1e7e2dd4908a369dc8734635702, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9689c61b42c94973964391a37edfb40c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8beb2789990c42cf8a8ad529b31e25c3] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=35.2 K 2024-11-15T04:40:18,667 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 20afd1e7e2dd4908a369dc8734635702, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731645615475 2024-11-15T04:40:18,676 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9689c61b42c94973964391a37edfb40c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731645615566 2024-11-15T04:40:18,676 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 8beb2789990c42cf8a8ad529b31e25c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731645616769 2024-11-15T04:40:18,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645678683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645678683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645678683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,692 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#C#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:18,693 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/2deaacbcb22c458b82dc8b9bb59c9184 is 50, key is test_row_0/C:col10/1731645616769/Put/seqid=0 2024-11-15T04:40:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645678692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741958_1134 (size=31058) 2024-11-15T04:40:18,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645678703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:18,714 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/26ca97a4adc84a9e8be873380a6c7851 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/26ca97a4adc84a9e8be873380a6c7851 2024-11-15T04:40:18,721 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/A of 465c1abe9e468bea9e6a5aa6fe32be31 into 26ca97a4adc84a9e8be873380a6c7851(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:18,721 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:18,722 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/A, priority=13, startTime=1731645618161; duration=0sec 2024-11-15T04:40:18,722 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:18,722 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:A 2024-11-15T04:40:18,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741959_1135 (size=12104) 2024-11-15T04:40:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T04:40:18,785 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/9bafd5140adb4ad58ea5bcafdc22700a 2024-11-15T04:40:18,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/6dc67441e0e74ec494fdd778fb807685 is 50, key is test_row_0/B:col10/1731645617166/Put/seqid=0 2024-11-15T04:40:18,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741960_1136 (size=12001) 2024-11-15T04:40:19,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645679003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645679003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645679003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645679003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645679013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,136 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/2deaacbcb22c458b82dc8b9bb59c9184 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2deaacbcb22c458b82dc8b9bb59c9184 2024-11-15T04:40:19,142 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/C of 465c1abe9e468bea9e6a5aa6fe32be31 into 2deaacbcb22c458b82dc8b9bb59c9184(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:19,142 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:19,142 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/C, priority=13, startTime=1731645618161; duration=0sec 2024-11-15T04:40:19,142 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:19,142 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:C 2024-11-15T04:40:19,213 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/6dc67441e0e74ec494fdd778fb807685 2024-11-15T04:40:19,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/ce09d5b6ae714b0d9a83b5981d05efb1 is 50, key is test_row_0/C:col10/1731645617166/Put/seqid=0 2024-11-15T04:40:19,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741961_1137 (size=12001) 2024-11-15T04:40:19,235 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/ce09d5b6ae714b0d9a83b5981d05efb1 2024-11-15T04:40:19,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/9bafd5140adb4ad58ea5bcafdc22700a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9bafd5140adb4ad58ea5bcafdc22700a 2024-11-15T04:40:19,254 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9bafd5140adb4ad58ea5bcafdc22700a, entries=150, sequenceid=77, filesize=30.2 K 2024-11-15T04:40:19,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/6dc67441e0e74ec494fdd778fb807685 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6dc67441e0e74ec494fdd778fb807685 2024-11-15T04:40:19,264 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6dc67441e0e74ec494fdd778fb807685, entries=150, sequenceid=77, filesize=11.7 K 2024-11-15T04:40:19,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/ce09d5b6ae714b0d9a83b5981d05efb1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ce09d5b6ae714b0d9a83b5981d05efb1 2024-11-15T04:40:19,276 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ce09d5b6ae714b0d9a83b5981d05efb1, entries=150, sequenceid=77, filesize=11.7 K 2024-11-15T04:40:19,277 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1032ms, sequenceid=77, compaction requested=false 2024-11-15T04:40:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-15T04:40:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-15T04:40:19,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-15T04:40:19,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6520 sec 2024-11-15T04:40:19,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.6580 sec 2024-11-15T04:40:19,515 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:19,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T04:40:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:19,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153bac23c61dbe428aa053fb7803ae07f5_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645619514/Put/seqid=0 2024-11-15T04:40:19,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645679556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741962_1138 (size=12154) 2024-11-15T04:40:19,580 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:19,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645679567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645679579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645679579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645679579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153bac23c61dbe428aa053fb7803ae07f5_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153bac23c61dbe428aa053fb7803ae07f5_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:19,599 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f14f615b0bbe41a181195396d8e12c5e, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:19,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f14f615b0bbe41a181195396d8e12c5e is 175, key is test_row_0/A:col10/1731645619514/Put/seqid=0 2024-11-15T04:40:19,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741963_1139 (size=30955) 2024-11-15T04:40:19,647 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f14f615b0bbe41a181195396d8e12c5e 2024-11-15T04:40:19,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/092fb273d6ed4730b26d957099e12af4 is 50, key is test_row_0/B:col10/1731645619514/Put/seqid=0 2024-11-15T04:40:19,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741964_1140 (size=12001) 2024-11-15T04:40:19,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/092fb273d6ed4730b26d957099e12af4 2024-11-15T04:40:19,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/f457e88170db4c5ba769894c1a01ce65 is 50, key is test_row_0/C:col10/1731645619514/Put/seqid=0 2024-11-15T04:40:19,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645679683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645679694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645679694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645679694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645679694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741965_1141 (size=12001) 2024-11-15T04:40:19,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/f457e88170db4c5ba769894c1a01ce65 2024-11-15T04:40:19,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f14f615b0bbe41a181195396d8e12c5e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f14f615b0bbe41a181195396d8e12c5e 2024-11-15T04:40:19,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f14f615b0bbe41a181195396d8e12c5e, entries=150, sequenceid=93, filesize=30.2 K 2024-11-15T04:40:19,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/092fb273d6ed4730b26d957099e12af4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/092fb273d6ed4730b26d957099e12af4 2024-11-15T04:40:19,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/092fb273d6ed4730b26d957099e12af4, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T04:40:19,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/f457e88170db4c5ba769894c1a01ce65 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/f457e88170db4c5ba769894c1a01ce65 2024-11-15T04:40:19,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/f457e88170db4c5ba769894c1a01ce65, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T04:40:19,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 465c1abe9e468bea9e6a5aa6fe32be31 in 252ms, sequenceid=93, compaction requested=true 2024-11-15T04:40:19,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:19,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:19,767 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:19,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:19,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:19,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:19,767 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:19,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:19,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:19,769 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:19,769 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/A is initiating minor compaction (all files) 2024-11-15T04:40:19,769 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/A in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:19,769 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/26ca97a4adc84a9e8be873380a6c7851, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9bafd5140adb4ad58ea5bcafdc22700a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f14f615b0bbe41a181195396d8e12c5e] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=90.8 K 2024-11-15T04:40:19,769 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:19,769 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/26ca97a4adc84a9e8be873380a6c7851, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9bafd5140adb4ad58ea5bcafdc22700a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f14f615b0bbe41a181195396d8e12c5e] 2024-11-15T04:40:19,770 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:19,770 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/B is initiating minor compaction (all files) 2024-11-15T04:40:19,770 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/B in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:19,770 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26ca97a4adc84a9e8be873380a6c7851, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731645616769 2024-11-15T04:40:19,770 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/1beba43a83d740d29f51cdd290b56b42, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6dc67441e0e74ec494fdd778fb807685, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/092fb273d6ed4730b26d957099e12af4] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=35.3 K 2024-11-15T04:40:19,770 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bafd5140adb4ad58ea5bcafdc22700a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731645617166 2024-11-15T04:40:19,770 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 1beba43a83d740d29f51cdd290b56b42, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731645616769 2024-11-15T04:40:19,771 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting f14f615b0bbe41a181195396d8e12c5e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645618359 2024-11-15T04:40:19,771 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc67441e0e74ec494fdd778fb807685, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731645617166 2024-11-15T04:40:19,771 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 092fb273d6ed4730b26d957099e12af4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645618359 2024-11-15T04:40:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-15T04:40:19,774 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:19,774 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:19,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T04:40:19,779 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:19,780 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:19,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T04:40:19,795 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:19,817 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#B#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:19,818 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/df313646453f4248a4f6ce75a3913bca is 50, key is test_row_0/B:col10/1731645619514/Put/seqid=0 2024-11-15T04:40:19,821 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411156b14fd305da04f04a22f4a9beae4387e_465c1abe9e468bea9e6a5aa6fe32be31 store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:19,825 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411156b14fd305da04f04a22f4a9beae4387e_465c1abe9e468bea9e6a5aa6fe32be31, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:19,826 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156b14fd305da04f04a22f4a9beae4387e_465c1abe9e468bea9e6a5aa6fe32be31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T04:40:19,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741967_1143 (size=4469) 2024-11-15T04:40:19,888 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#A#compaction#120 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:19,888 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/57f95b0b344941feb1656f9a28f0610d is 175, key is test_row_0/A:col10/1731645619514/Put/seqid=0 2024-11-15T04:40:19,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741966_1142 (size=12207) 2024-11-15T04:40:19,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:19,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T04:40:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:19,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741968_1144 (size=31161) 2024-11-15T04:40:19,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150a2639a9bc04411188d608ea7a52e9cc_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645619571/Put/seqid=0 2024-11-15T04:40:19,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741969_1145 (size=14594) 2024-11-15T04:40:19,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645679923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645679923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645679925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T04:40:19,937 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:19,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:19,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:19,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:19,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:19,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:19,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645679931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:19,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645679933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:19,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150a2639a9bc04411188d608ea7a52e9cc_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150a2639a9bc04411188d608ea7a52e9cc_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:19,948 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/6b723d7f763f44728000ddbdc74332df, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:19,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/6b723d7f763f44728000ddbdc74332df is 175, key is test_row_0/A:col10/1731645619571/Put/seqid=0 2024-11-15T04:40:19,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741970_1146 (size=39549) 2024-11-15T04:40:20,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645680042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645680042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645680042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645680043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645680043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T04:40:20,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T04:40:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T04:40:20,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:20,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645680252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645680253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645680253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645680253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645680253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,302 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/df313646453f4248a4f6ce75a3913bca as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/df313646453f4248a4f6ce75a3913bca 2024-11-15T04:40:20,315 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/B of 465c1abe9e468bea9e6a5aa6fe32be31 into df313646453f4248a4f6ce75a3913bca(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:20,315 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:20,315 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/B, priority=13, startTime=1731645619767; duration=0sec 2024-11-15T04:40:20,315 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:20,315 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:B 2024-11-15T04:40:20,315 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:20,318 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:20,318 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/C is initiating minor compaction (all files) 2024-11-15T04:40:20,318 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/C in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,318 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2deaacbcb22c458b82dc8b9bb59c9184, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ce09d5b6ae714b0d9a83b5981d05efb1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/f457e88170db4c5ba769894c1a01ce65] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=35.3 K 2024-11-15T04:40:20,319 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 2deaacbcb22c458b82dc8b9bb59c9184, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731645616769 2024-11-15T04:40:20,320 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting ce09d5b6ae714b0d9a83b5981d05efb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731645617166 2024-11-15T04:40:20,321 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting f457e88170db4c5ba769894c1a01ce65, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645618359 2024-11-15T04:40:20,324 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/57f95b0b344941feb1656f9a28f0610d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/57f95b0b344941feb1656f9a28f0610d 2024-11-15T04:40:20,332 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/A of 465c1abe9e468bea9e6a5aa6fe32be31 into 57f95b0b344941feb1656f9a28f0610d(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:20,332 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:20,332 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/A, priority=13, startTime=1731645619767; duration=0sec 2024-11-15T04:40:20,332 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:20,332 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:A 2024-11-15T04:40:20,333 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:20,333 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/5ca62f0085574f7cb4690af1a7fb25a3 is 50, key is test_row_0/C:col10/1731645619514/Put/seqid=0 2024-11-15T04:40:20,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741971_1147 (size=12207) 2024-11-15T04:40:20,367 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/6b723d7f763f44728000ddbdc74332df 2024-11-15T04:40:20,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/bc88b36bdf6e49a6b21b5c046bac03ee is 50, key is test_row_0/B:col10/1731645619571/Put/seqid=0 2024-11-15T04:40:20,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741972_1148 (size=12001) 2024-11-15T04:40:20,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/bc88b36bdf6e49a6b21b5c046bac03ee 2024-11-15T04:40:20,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T04:40:20,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:20,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T04:40:20,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/081ef298f95d4b18a30b99ab1a909695 is 50, key is test_row_0/C:col10/1731645619571/Put/seqid=0 2024-11-15T04:40:20,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741973_1149 (size=12001) 2024-11-15T04:40:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645680590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645680586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645680591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645680591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645680591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T04:40:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:20,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:20,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:20,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T04:40:20,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:20,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:20,780 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/5ca62f0085574f7cb4690af1a7fb25a3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5ca62f0085574f7cb4690af1a7fb25a3 2024-11-15T04:40:20,798 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/C of 465c1abe9e468bea9e6a5aa6fe32be31 into 5ca62f0085574f7cb4690af1a7fb25a3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:20,798 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:20,798 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/C, priority=13, startTime=1731645619767; duration=0sec 2024-11-15T04:40:20,798 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:20,798 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:C 2024-11-15T04:40:20,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/081ef298f95d4b18a30b99ab1a909695 2024-11-15T04:40:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/6b723d7f763f44728000ddbdc74332df as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/6b723d7f763f44728000ddbdc74332df 2024-11-15T04:40:20,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/6b723d7f763f44728000ddbdc74332df, entries=200, sequenceid=117, filesize=38.6 K 2024-11-15T04:40:20,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/bc88b36bdf6e49a6b21b5c046bac03ee as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/bc88b36bdf6e49a6b21b5c046bac03ee 2024-11-15T04:40:20,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/bc88b36bdf6e49a6b21b5c046bac03ee, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T04:40:20,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/081ef298f95d4b18a30b99ab1a909695 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/081ef298f95d4b18a30b99ab1a909695 2024-11-15T04:40:20,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/081ef298f95d4b18a30b99ab1a909695, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T04:40:20,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 465c1abe9e468bea9e6a5aa6fe32be31 in 964ms, sequenceid=117, compaction requested=false 2024-11-15T04:40:20,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:20,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T04:40:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:20,903 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T04:40:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T04:40:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158c66426e79004efc927b8fcd7a3b5a49_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645619929/Put/seqid=0 2024-11-15T04:40:20,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741974_1150 (size=12154) 2024-11-15T04:40:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:21,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:21,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645681145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645681148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645681150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645681151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645681153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645681263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645681263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645681263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645681263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645681264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:21,341 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158c66426e79004efc927b8fcd7a3b5a49_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158c66426e79004efc927b8fcd7a3b5a49_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:21,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/04418f1e05504e788ecf5c0535517df9, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:21,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/04418f1e05504e788ecf5c0535517df9 is 175, key is test_row_0/A:col10/1731645619929/Put/seqid=0 2024-11-15T04:40:21,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741975_1151 (size=30955) 2024-11-15T04:40:21,366 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/04418f1e05504e788ecf5c0535517df9 2024-11-15T04:40:21,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/56a4f8927dc943a2ad0de18c9e3e02c9 is 50, key is test_row_0/B:col10/1731645619929/Put/seqid=0 2024-11-15T04:40:21,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741976_1152 (size=12001) 2024-11-15T04:40:21,427 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/56a4f8927dc943a2ad0de18c9e3e02c9 2024-11-15T04:40:21,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/c74f4c3978134a5383932829e44aad1b is 50, key is test_row_0/C:col10/1731645619929/Put/seqid=0 2024-11-15T04:40:21,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741977_1153 (size=12001) 2024-11-15T04:40:21,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645681483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645681483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645681483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645681483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645681483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645681793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645681793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645681793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645681793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645681793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:21,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:21,851 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/c74f4c3978134a5383932829e44aad1b 2024-11-15T04:40:21,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/04418f1e05504e788ecf5c0535517df9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/04418f1e05504e788ecf5c0535517df9 2024-11-15T04:40:21,861 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/04418f1e05504e788ecf5c0535517df9, entries=150, sequenceid=132, filesize=30.2 K 2024-11-15T04:40:21,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/56a4f8927dc943a2ad0de18c9e3e02c9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/56a4f8927dc943a2ad0de18c9e3e02c9 2024-11-15T04:40:21,866 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/56a4f8927dc943a2ad0de18c9e3e02c9, entries=150, sequenceid=132, filesize=11.7 K 2024-11-15T04:40:21,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/c74f4c3978134a5383932829e44aad1b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/c74f4c3978134a5383932829e44aad1b 2024-11-15T04:40:21,873 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/c74f4c3978134a5383932829e44aad1b, entries=150, sequenceid=132, filesize=11.7 K 2024-11-15T04:40:21,874 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 465c1abe9e468bea9e6a5aa6fe32be31 in 971ms, sequenceid=132, compaction requested=true 2024-11-15T04:40:21,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:21,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:21,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T04:40:21,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T04:40:21,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T04:40:21,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0960 sec 2024-11-15T04:40:21,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.1040 sec 2024-11-15T04:40:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T04:40:21,923 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:21,923 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:21,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T04:40:21,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T04:40:21,925 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:21,926 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:21,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T04:40:22,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T04:40:22,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:22,081 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:40:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155b5dbcebe39642c6b9e4246bd4fa5f93_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645621149/Put/seqid=0 2024-11-15T04:40:22,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741978_1154 (size=12304) 2024-11-15T04:40:22,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:22,170 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155b5dbcebe39642c6b9e4246bd4fa5f93_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155b5dbcebe39642c6b9e4246bd4fa5f93_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:22,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ede1538d7c94496aa6f7e319795333ce, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:22,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ede1538d7c94496aa6f7e319795333ce is 175, key is test_row_0/A:col10/1731645621149/Put/seqid=0 2024-11-15T04:40:22,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741979_1155 (size=31105) 2024-11-15T04:40:22,213 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ede1538d7c94496aa6f7e319795333ce 2024-11-15T04:40:22,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/e9b05e0f96814931bba382c0263179ab is 50, key is test_row_0/B:col10/1731645621149/Put/seqid=0 2024-11-15T04:40:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T04:40:22,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741980_1156 (size=12151) 2024-11-15T04:40:22,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:22,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645682315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645682317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645682322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645682323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645682324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645682432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645682433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645682433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645682433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645682443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T04:40:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645682643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645682643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645682643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645682643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645682652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,666 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/e9b05e0f96814931bba382c0263179ab 2024-11-15T04:40:22,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/9b4668b2bd7345ab9e99f4b1c298a19e is 50, key is test_row_0/C:col10/1731645621149/Put/seqid=0 2024-11-15T04:40:22,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741981_1157 (size=12151) 2024-11-15T04:40:22,742 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/9b4668b2bd7345ab9e99f4b1c298a19e 2024-11-15T04:40:22,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ede1538d7c94496aa6f7e319795333ce as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ede1538d7c94496aa6f7e319795333ce 2024-11-15T04:40:22,771 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ede1538d7c94496aa6f7e319795333ce, entries=150, sequenceid=157, filesize=30.4 K 2024-11-15T04:40:22,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/e9b05e0f96814931bba382c0263179ab as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e9b05e0f96814931bba382c0263179ab 2024-11-15T04:40:22,787 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e9b05e0f96814931bba382c0263179ab, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T04:40:22,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/9b4668b2bd7345ab9e99f4b1c298a19e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9b4668b2bd7345ab9e99f4b1c298a19e 2024-11-15T04:40:22,805 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9b4668b2bd7345ab9e99f4b1c298a19e, entries=150, sequenceid=157, filesize=11.9 K 2024-11-15T04:40:22,806 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 465c1abe9e468bea9e6a5aa6fe32be31 in 725ms, sequenceid=157, compaction requested=true 2024-11-15T04:40:22,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:22,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:22,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T04:40:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T04:40:22,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T04:40:22,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 885 msec 2024-11-15T04:40:22,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 892 msec 2024-11-15T04:40:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:22,957 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:22,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T04:40:22,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153dad663b473642798793b82fe235b160_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645622315/Put/seqid=0 2024-11-15T04:40:22,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741982_1158 (size=14794) 2024-11-15T04:40:22,990 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:22,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645682988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645682990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645682994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645682995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:22,999 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153dad663b473642798793b82fe235b160_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153dad663b473642798793b82fe235b160_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:23,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645682999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,001 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f107f12288af40be9b2fa9fa7dbd4e5e, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:23,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f107f12288af40be9b2fa9fa7dbd4e5e is 175, key is test_row_0/A:col10/1731645622315/Put/seqid=0 2024-11-15T04:40:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741983_1159 (size=39749) 2024-11-15T04:40:23,017 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f107f12288af40be9b2fa9fa7dbd4e5e 2024-11-15T04:40:23,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/15a04b33486142e095d34e177071a724 is 50, key is test_row_0/B:col10/1731645622315/Put/seqid=0 2024-11-15T04:40:23,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741984_1160 (size=12151) 2024-11-15T04:40:23,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/15a04b33486142e095d34e177071a724 2024-11-15T04:40:23,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/229326c7741041d59851182ca2e1885b is 50, key is test_row_0/C:col10/1731645622315/Put/seqid=0 2024-11-15T04:40:23,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741985_1161 (size=12151) 2024-11-15T04:40:23,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/229326c7741041d59851182ca2e1885b 2024-11-15T04:40:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T04:40:23,063 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:23,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T04:40:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T04:40:23,066 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:23,066 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:23,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:23,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f107f12288af40be9b2fa9fa7dbd4e5e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f107f12288af40be9b2fa9fa7dbd4e5e 2024-11-15T04:40:23,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f107f12288af40be9b2fa9fa7dbd4e5e, entries=200, sequenceid=170, filesize=38.8 K 2024-11-15T04:40:23,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/15a04b33486142e095d34e177071a724 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/15a04b33486142e095d34e177071a724 2024-11-15T04:40:23,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/15a04b33486142e095d34e177071a724, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T04:40:23,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/229326c7741041d59851182ca2e1885b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/229326c7741041d59851182ca2e1885b 2024-11-15T04:40:23,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/229326c7741041d59851182ca2e1885b, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T04:40:23,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 465c1abe9e468bea9e6a5aa6fe32be31 in 139ms, sequenceid=170, compaction requested=true 2024-11-15T04:40:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:23,096 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T04:40:23,098 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172519 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T04:40:23,098 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/A is initiating minor compaction (all files) 2024-11-15T04:40:23,098 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/A in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,098 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/57f95b0b344941feb1656f9a28f0610d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/6b723d7f763f44728000ddbdc74332df, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/04418f1e05504e788ecf5c0535517df9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ede1538d7c94496aa6f7e319795333ce, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f107f12288af40be9b2fa9fa7dbd4e5e] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=168.5 K 2024-11-15T04:40:23,098 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,098 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/57f95b0b344941feb1656f9a28f0610d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/6b723d7f763f44728000ddbdc74332df, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/04418f1e05504e788ecf5c0535517df9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ede1538d7c94496aa6f7e319795333ce, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f107f12288af40be9b2fa9fa7dbd4e5e] 2024-11-15T04:40:23,100 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57f95b0b344941feb1656f9a28f0610d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645618359 2024-11-15T04:40:23,101 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b723d7f763f44728000ddbdc74332df, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645619550 2024-11-15T04:40:23,102 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04418f1e05504e788ecf5c0535517df9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731645619921 2024-11-15T04:40:23,102 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting ede1538d7c94496aa6f7e319795333ce, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731645621136 2024-11-15T04:40:23,102 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting f107f12288af40be9b2fa9fa7dbd4e5e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731645622315 2024-11-15T04:40:23,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:23,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:23,106 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T04:40:23,111 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60511 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T04:40:23,111 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/B is initiating minor compaction (all files) 2024-11-15T04:40:23,111 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/B in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,112 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/df313646453f4248a4f6ce75a3913bca, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/bc88b36bdf6e49a6b21b5c046bac03ee, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/56a4f8927dc943a2ad0de18c9e3e02c9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e9b05e0f96814931bba382c0263179ab, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/15a04b33486142e095d34e177071a724] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=59.1 K 2024-11-15T04:40:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:23,113 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting df313646453f4248a4f6ce75a3913bca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645618359 2024-11-15T04:40:23,114 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting bc88b36bdf6e49a6b21b5c046bac03ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645619550 2024-11-15T04:40:23,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:23,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:23,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:23,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:23,115 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:23,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T04:40:23,116 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a4f8927dc943a2ad0de18c9e3e02c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731645619921 2024-11-15T04:40:23,117 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b05e0f96814931bba382c0263179ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731645621136 2024-11-15T04:40:23,118 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 15a04b33486142e095d34e177071a724, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731645622315 2024-11-15T04:40:23,130 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:23,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645683126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115def97644733d457992b3c813eece6c8b_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645623111/Put/seqid=0 2024-11-15T04:40:23,156 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411157515e9e84d594183b28ca74fd96124bb_465c1abe9e468bea9e6a5aa6fe32be31 store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:23,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645683130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645683139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645683139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645683145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,163 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411157515e9e84d594183b28ca74fd96124bb_465c1abe9e468bea9e6a5aa6fe32be31, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:23,163 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157515e9e84d594183b28ca74fd96124bb_465c1abe9e468bea9e6a5aa6fe32be31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T04:40:23,186 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#B#compaction#137 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:23,187 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/fcda5f8773cf4027beeeaaf31281294a is 50, key is test_row_0/B:col10/1731645622315/Put/seqid=0 2024-11-15T04:40:23,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:23,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:23,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741986_1162 (size=14794) 2024-11-15T04:40:23,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645683253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645683263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645683263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645683263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741987_1163 (size=4469) 2024-11-15T04:40:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645683264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741988_1164 (size=12527) 2024-11-15T04:40:23,291 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/fcda5f8773cf4027beeeaaf31281294a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/fcda5f8773cf4027beeeaaf31281294a 2024-11-15T04:40:23,298 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/B of 465c1abe9e468bea9e6a5aa6fe32be31 into fcda5f8773cf4027beeeaaf31281294a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:23,298 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:23,298 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/B, priority=11, startTime=1731645623106; duration=0sec 2024-11-15T04:40:23,298 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:23,298 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:B 2024-11-15T04:40:23,298 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T04:40:23,300 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60511 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T04:40:23,300 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/C is initiating minor compaction (all files) 2024-11-15T04:40:23,300 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/C in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,301 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5ca62f0085574f7cb4690af1a7fb25a3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/081ef298f95d4b18a30b99ab1a909695, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/c74f4c3978134a5383932829e44aad1b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9b4668b2bd7345ab9e99f4b1c298a19e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/229326c7741041d59851182ca2e1885b] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=59.1 K 2024-11-15T04:40:23,301 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca62f0085574f7cb4690af1a7fb25a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645618359 2024-11-15T04:40:23,302 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 081ef298f95d4b18a30b99ab1a909695, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645619550 2024-11-15T04:40:23,302 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c74f4c3978134a5383932829e44aad1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731645619921 2024-11-15T04:40:23,303 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b4668b2bd7345ab9e99f4b1c298a19e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731645621136 2024-11-15T04:40:23,304 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 229326c7741041d59851182ca2e1885b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731645622315 2024-11-15T04:40:23,334 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#C#compaction#138 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:23,335 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/ddbbf08e9c404d2a966d38cc1001d566 is 50, key is test_row_0/C:col10/1731645622315/Put/seqid=0 2024-11-15T04:40:23,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:23,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:23,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T04:40:23,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741989_1165 (size=12527) 2024-11-15T04:40:23,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645683474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645683474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645683474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645683474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645683485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:23,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:23,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,637 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:23,643 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115def97644733d457992b3c813eece6c8b_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115def97644733d457992b3c813eece6c8b_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:23,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/2e89413a19e2421096bb3f4c12fccbb5, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:23,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/2e89413a19e2421096bb3f4c12fccbb5 is 175, key is test_row_0/A:col10/1731645623111/Put/seqid=0 2024-11-15T04:40:23,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741990_1166 (size=39749) 2024-11-15T04:40:23,671 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#A#compaction#135 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:23,672 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/35bc7e44dcd9480cb4fb6415a8b424f1 is 175, key is test_row_0/A:col10/1731645622315/Put/seqid=0 2024-11-15T04:40:23,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:23,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:23,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T04:40:23,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741991_1167 (size=31481) 2024-11-15T04:40:23,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645683793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,801 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/ddbbf08e9c404d2a966d38cc1001d566 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ddbbf08e9c404d2a966d38cc1001d566 2024-11-15T04:40:23,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645683793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645683793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645683793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645683807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:23,817 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/C of 465c1abe9e468bea9e6a5aa6fe32be31 into ddbbf08e9c404d2a966d38cc1001d566(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:23,817 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:23,818 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/C, priority=11, startTime=1731645623115; duration=0sec 2024-11-15T04:40:23,818 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:23,818 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:C 2024-11-15T04:40:23,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:23,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:23,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:23,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/2e89413a19e2421096bb3f4c12fccbb5 2024-11-15T04:40:24,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/de53ad56b7954f1086a085126a1911c2 is 50, key is test_row_0/B:col10/1731645623111/Put/seqid=0 2024-11-15T04:40:24,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741992_1168 (size=12151) 2024-11-15T04:40:24,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/de53ad56b7954f1086a085126a1911c2 2024-11-15T04:40:24,107 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/35bc7e44dcd9480cb4fb6415a8b424f1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/35bc7e44dcd9480cb4fb6415a8b424f1 2024-11-15T04:40:24,117 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/A of 465c1abe9e468bea9e6a5aa6fe32be31 into 35bc7e44dcd9480cb4fb6415a8b424f1(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-15T04:40:24,117 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:24,117 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/A, priority=11, startTime=1731645623096; duration=1sec 2024-11-15T04:40:24,118 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:24,118 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:A 2024-11-15T04:40:24,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/36a146b810b8494f80ce58859749831d is 50, key is test_row_0/C:col10/1731645623111/Put/seqid=0 2024-11-15T04:40:24,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:24,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:24,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T04:40:24,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741993_1169 (size=12151) 2024-11-15T04:40:24,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/36a146b810b8494f80ce58859749831d 2024-11-15T04:40:24,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/2e89413a19e2421096bb3f4c12fccbb5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/2e89413a19e2421096bb3f4c12fccbb5 2024-11-15T04:40:24,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/2e89413a19e2421096bb3f4c12fccbb5, entries=200, sequenceid=197, filesize=38.8 K 2024-11-15T04:40:24,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/de53ad56b7954f1086a085126a1911c2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/de53ad56b7954f1086a085126a1911c2 2024-11-15T04:40:24,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/de53ad56b7954f1086a085126a1911c2, entries=150, sequenceid=197, filesize=11.9 K 2024-11-15T04:40:24,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/36a146b810b8494f80ce58859749831d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/36a146b810b8494f80ce58859749831d 2024-11-15T04:40:24,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/36a146b810b8494f80ce58859749831d, entries=150, sequenceid=197, filesize=11.9 K 2024-11-15T04:40:24,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1168ms, sequenceid=197, compaction requested=false 2024-11-15T04:40:24,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:24,315 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:24,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:40:24,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:24,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:24,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a13fbe82320948cf9868f191f1d0086d_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645623127/Put/seqid=0 2024-11-15T04:40:24,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645684368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645684368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645684368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645684370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645684370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741994_1170 (size=14794) 2024-11-15T04:40:24,390 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:24,397 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a13fbe82320948cf9868f191f1d0086d_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a13fbe82320948cf9868f191f1d0086d_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:24,398 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/dbd34cea6bbb469d92bf3347d726ad76, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:24,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/dbd34cea6bbb469d92bf3347d726ad76 is 175, key is test_row_0/A:col10/1731645623127/Put/seqid=0 2024-11-15T04:40:24,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741995_1171 (size=39749) 2024-11-15T04:40:24,415 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/dbd34cea6bbb469d92bf3347d726ad76 2024-11-15T04:40:24,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/6d4b936087624f868d5c356c840d631c is 50, key is test_row_0/B:col10/1731645623127/Put/seqid=0 2024-11-15T04:40:24,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741996_1172 (size=12151) 2024-11-15T04:40:24,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/6d4b936087624f868d5c356c840d631c 2024-11-15T04:40:24,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/438d5b28b7c24358a202b3111510c5ed is 50, key is test_row_0/C:col10/1731645623127/Put/seqid=0 2024-11-15T04:40:24,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741997_1173 (size=12151) 2024-11-15T04:40:24,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645684483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645684483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645684483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645684483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645684483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:24,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:24,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:24,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:24,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645684692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645684692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645684692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645684693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645684693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:24,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:24,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:24,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/438d5b28b7c24358a202b3111510c5ed 2024-11-15T04:40:24,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/dbd34cea6bbb469d92bf3347d726ad76 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/dbd34cea6bbb469d92bf3347d726ad76 2024-11-15T04:40:24,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/dbd34cea6bbb469d92bf3347d726ad76, entries=200, sequenceid=212, filesize=38.8 K 2024-11-15T04:40:24,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/6d4b936087624f868d5c356c840d631c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6d4b936087624f868d5c356c840d631c 2024-11-15T04:40:24,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6d4b936087624f868d5c356c840d631c, entries=150, sequenceid=212, filesize=11.9 K 2024-11-15T04:40:24,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/438d5b28b7c24358a202b3111510c5ed as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/438d5b28b7c24358a202b3111510c5ed 2024-11-15T04:40:24,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/438d5b28b7c24358a202b3111510c5ed, entries=150, sequenceid=212, filesize=11.9 K 2024-11-15T04:40:24,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 465c1abe9e468bea9e6a5aa6fe32be31 in 587ms, sequenceid=212, compaction requested=true 2024-11-15T04:40:24,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:24,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:24,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:24,903 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:24,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:24,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:24,903 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:24,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:24,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:24,906 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:24,906 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:24,906 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/B is initiating minor compaction (all files) 2024-11-15T04:40:24,906 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/A is initiating minor compaction (all files) 2024-11-15T04:40:24,906 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/B in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,906 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/A in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,906 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/35bc7e44dcd9480cb4fb6415a8b424f1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/2e89413a19e2421096bb3f4c12fccbb5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/dbd34cea6bbb469d92bf3347d726ad76] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=108.4 K 2024-11-15T04:40:24,906 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/fcda5f8773cf4027beeeaaf31281294a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/de53ad56b7954f1086a085126a1911c2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6d4b936087624f868d5c356c840d631c] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=36.0 K 2024-11-15T04:40:24,906 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,906 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/35bc7e44dcd9480cb4fb6415a8b424f1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/2e89413a19e2421096bb3f4c12fccbb5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/dbd34cea6bbb469d92bf3347d726ad76] 2024-11-15T04:40:24,907 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35bc7e44dcd9480cb4fb6415a8b424f1, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731645622315 2024-11-15T04:40:24,907 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting fcda5f8773cf4027beeeaaf31281294a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731645622315 2024-11-15T04:40:24,907 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e89413a19e2421096bb3f4c12fccbb5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731645623108 2024-11-15T04:40:24,907 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting de53ad56b7954f1086a085126a1911c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731645623111 2024-11-15T04:40:24,907 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbd34cea6bbb469d92bf3347d726ad76, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731645623124 2024-11-15T04:40:24,908 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4b936087624f868d5c356c840d631c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731645623127 2024-11-15T04:40:24,914 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:24,915 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#B#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:24,916 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/c6fe827cd2ad435184a6c37033dc2f78 is 50, key is test_row_0/B:col10/1731645623127/Put/seqid=0 2024-11-15T04:40:24,919 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115522df3199d274d9990b907a77cdfed0d_465c1abe9e468bea9e6a5aa6fe32be31 store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:24,922 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115522df3199d274d9990b907a77cdfed0d_465c1abe9e468bea9e6a5aa6fe32be31, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:24,922 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115522df3199d274d9990b907a77cdfed0d_465c1abe9e468bea9e6a5aa6fe32be31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:24,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741998_1174 (size=12629) 2024-11-15T04:40:24,939 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/c6fe827cd2ad435184a6c37033dc2f78 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c6fe827cd2ad435184a6c37033dc2f78 2024-11-15T04:40:24,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741999_1175 (size=4469) 2024-11-15T04:40:24,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T04:40:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,947 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/B of 465c1abe9e468bea9e6a5aa6fe32be31 into c6fe827cd2ad435184a6c37033dc2f78(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:24,947 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T04:40:24,947 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:24,947 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/B, priority=13, startTime=1731645624903; duration=0sec 2024-11-15T04:40:24,948 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:24,948 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:B 2024-11-15T04:40:24,948 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:24,950 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:24,950 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/C is initiating minor compaction (all files) 2024-11-15T04:40:24,950 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/C in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:24,951 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ddbbf08e9c404d2a966d38cc1001d566, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/36a146b810b8494f80ce58859749831d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/438d5b28b7c24358a202b3111510c5ed] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=36.0 K 2024-11-15T04:40:24,951 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting ddbbf08e9c404d2a966d38cc1001d566, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731645622315 2024-11-15T04:40:24,951 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 36a146b810b8494f80ce58859749831d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731645623111 2024-11-15T04:40:24,952 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 438d5b28b7c24358a202b3111510c5ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731645623127 2024-11-15T04:40:24,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d2297fc41092413b84eacfb88a927b2c_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645624368/Put/seqid=0 2024-11-15T04:40:24,966 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:24,966 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/6e9654ca510d45b1863a07272badd03f is 50, key is test_row_0/C:col10/1731645623127/Put/seqid=0 2024-11-15T04:40:24,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742000_1176 (size=12304) 2024-11-15T04:40:24,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742001_1177 (size=12629) 2024-11-15T04:40:25,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:25,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645685013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645685014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645685016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645685018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645685018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645685123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645685123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645685123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645685123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645685123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T04:40:25,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645685333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645685333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645685333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645685333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645685333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,344 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#A#compaction#145 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:25,345 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/61abc808739e4cfca587bcf25b007d73 is 175, key is test_row_0/A:col10/1731645623127/Put/seqid=0 2024-11-15T04:40:25,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742002_1178 (size=31583) 2024-11-15T04:40:25,359 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/61abc808739e4cfca587bcf25b007d73 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/61abc808739e4cfca587bcf25b007d73 2024-11-15T04:40:25,364 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/A of 465c1abe9e468bea9e6a5aa6fe32be31 into 61abc808739e4cfca587bcf25b007d73(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:25,364 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:25,364 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/A, priority=13, startTime=1731645624903; duration=0sec 2024-11-15T04:40:25,364 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:25,364 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:A 2024-11-15T04:40:25,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:25,378 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d2297fc41092413b84eacfb88a927b2c_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d2297fc41092413b84eacfb88a927b2c_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:25,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/0f7d80838c1a4e32a7867716a060d897, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:25,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/0f7d80838c1a4e32a7867716a060d897 is 175, key is test_row_0/A:col10/1731645624368/Put/seqid=0 2024-11-15T04:40:25,381 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/6e9654ca510d45b1863a07272badd03f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/6e9654ca510d45b1863a07272badd03f 2024-11-15T04:40:25,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742003_1179 (size=31105) 2024-11-15T04:40:25,387 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/0f7d80838c1a4e32a7867716a060d897 2024-11-15T04:40:25,387 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/C of 465c1abe9e468bea9e6a5aa6fe32be31 into 6e9654ca510d45b1863a07272badd03f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:25,387 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:25,387 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/C, priority=13, startTime=1731645624903; duration=0sec 2024-11-15T04:40:25,387 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:25,387 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:C 2024-11-15T04:40:25,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/d94bfaa4bd764fecba638c2d1a7caa95 is 50, key is test_row_0/B:col10/1731645624368/Put/seqid=0 2024-11-15T04:40:25,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742004_1180 (size=12151) 2024-11-15T04:40:25,584 INFO [master/26747e6ada4f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T04:40:25,584 INFO [master/26747e6ada4f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T04:40:25,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645685643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645685653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645685653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645685653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645685653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:25,799 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/d94bfaa4bd764fecba638c2d1a7caa95 2024-11-15T04:40:25,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/391fa96d1d8c462688e162d4ebbe1758 is 50, key is test_row_0/C:col10/1731645624368/Put/seqid=0 2024-11-15T04:40:25,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742005_1181 (size=12151) 2024-11-15T04:40:26,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645686153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:26,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645686162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645686162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645686162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:26,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645686162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:26,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:26,226 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/391fa96d1d8c462688e162d4ebbe1758 2024-11-15T04:40:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/0f7d80838c1a4e32a7867716a060d897 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/0f7d80838c1a4e32a7867716a060d897 2024-11-15T04:40:26,240 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/0f7d80838c1a4e32a7867716a060d897, entries=150, sequenceid=237, filesize=30.4 K 2024-11-15T04:40:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/d94bfaa4bd764fecba638c2d1a7caa95 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/d94bfaa4bd764fecba638c2d1a7caa95 2024-11-15T04:40:26,246 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/d94bfaa4bd764fecba638c2d1a7caa95, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T04:40:26,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/391fa96d1d8c462688e162d4ebbe1758 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/391fa96d1d8c462688e162d4ebbe1758 2024-11-15T04:40:26,251 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/391fa96d1d8c462688e162d4ebbe1758, entries=150, sequenceid=237, filesize=11.9 K 2024-11-15T04:40:26,252 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1305ms, sequenceid=237, compaction requested=false 2024-11-15T04:40:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T04:40:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T04:40:26,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T04:40:26,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1870 sec 2024-11-15T04:40:26,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 3.1930 sec 2024-11-15T04:40:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:27,166 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:27,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T04:40:27,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d81ebbac587145b5b4b8abe574363527_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645625010/Put/seqid=0 2024-11-15T04:40:27,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742006_1182 (size=14794) 2024-11-15T04:40:27,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645687191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645687195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645687196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645687197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645687197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T04:40:27,233 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:27,234 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T04:40:27,235 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T04:40:27,236 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:27,236 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:27,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645687303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645687303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645687303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645687313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645687313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T04:40:27,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:27,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645687513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645687513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645687513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645687522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645687522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T04:40:27,580 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:27,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d81ebbac587145b5b4b8abe574363527_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d81ebbac587145b5b4b8abe574363527_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:27,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/8270e2c29b054eb1ad65e4e590fabb61, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:27,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/8270e2c29b054eb1ad65e4e590fabb61 is 175, key is test_row_0/A:col10/1731645625010/Put/seqid=0 2024-11-15T04:40:27,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742007_1183 (size=39749) 2024-11-15T04:40:27,591 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/8270e2c29b054eb1ad65e4e590fabb61 2024-11-15T04:40:27,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/9ce1cf7a37dc4c1c89a95a68a277d8f3 is 50, key is test_row_0/B:col10/1731645625010/Put/seqid=0 2024-11-15T04:40:27,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742008_1184 (size=12151) 2024-11-15T04:40:27,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/9ce1cf7a37dc4c1c89a95a68a277d8f3 2024-11-15T04:40:27,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/32c2f33265bd487f9377e06f5929b06e is 50, key is test_row_0/C:col10/1731645625010/Put/seqid=0 2024-11-15T04:40:27,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742009_1185 (size=12151) 2024-11-15T04:40:27,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/32c2f33265bd487f9377e06f5929b06e 2024-11-15T04:40:27,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/8270e2c29b054eb1ad65e4e590fabb61 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/8270e2c29b054eb1ad65e4e590fabb61 2024-11-15T04:40:27,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/8270e2c29b054eb1ad65e4e590fabb61, entries=200, sequenceid=253, filesize=38.8 K 2024-11-15T04:40:27,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/9ce1cf7a37dc4c1c89a95a68a277d8f3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9ce1cf7a37dc4c1c89a95a68a277d8f3 2024-11-15T04:40:27,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9ce1cf7a37dc4c1c89a95a68a277d8f3, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T04:40:27,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:27,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/32c2f33265bd487f9377e06f5929b06e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/32c2f33265bd487f9377e06f5929b06e 2024-11-15T04:40:27,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/32c2f33265bd487f9377e06f5929b06e, entries=150, sequenceid=253, filesize=11.9 K 2024-11-15T04:40:27,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 465c1abe9e468bea9e6a5aa6fe32be31 in 544ms, sequenceid=253, compaction requested=true 2024-11-15T04:40:27,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:27,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:27,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:27,710 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:27,710 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:27,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:27,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:27,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:27,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:27,712 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:27,712 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/A is initiating minor compaction (all files) 2024-11-15T04:40:27,712 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/A in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,712 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/61abc808739e4cfca587bcf25b007d73, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/0f7d80838c1a4e32a7867716a060d897, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/8270e2c29b054eb1ad65e4e590fabb61] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=100.0 K 2024-11-15T04:40:27,712 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,712 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/61abc808739e4cfca587bcf25b007d73, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/0f7d80838c1a4e32a7867716a060d897, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/8270e2c29b054eb1ad65e4e590fabb61] 2024-11-15T04:40:27,713 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:27,713 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/B is initiating minor compaction (all files) 2024-11-15T04:40:27,713 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/B in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,713 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c6fe827cd2ad435184a6c37033dc2f78, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/d94bfaa4bd764fecba638c2d1a7caa95, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9ce1cf7a37dc4c1c89a95a68a277d8f3] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=36.1 K 2024-11-15T04:40:27,714 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61abc808739e4cfca587bcf25b007d73, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731645623127 2024-11-15T04:40:27,714 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c6fe827cd2ad435184a6c37033dc2f78, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731645623127 2024-11-15T04:40:27,714 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f7d80838c1a4e32a7867716a060d897, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731645624364 2024-11-15T04:40:27,714 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting d94bfaa4bd764fecba638c2d1a7caa95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731645624364 2024-11-15T04:40:27,715 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ce1cf7a37dc4c1c89a95a68a277d8f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731645625010 2024-11-15T04:40:27,715 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8270e2c29b054eb1ad65e4e590fabb61, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731645625010 2024-11-15T04:40:27,727 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#B#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:27,727 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/9be7caf16a6f4c0c908f3bff8d40b5bc is 50, key is test_row_0/B:col10/1731645625010/Put/seqid=0 2024-11-15T04:40:27,728 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:27,732 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115fd0d0b67d7284fffbc154f0ff28933aa_465c1abe9e468bea9e6a5aa6fe32be31 store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:27,734 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115fd0d0b67d7284fffbc154f0ff28933aa_465c1abe9e468bea9e6a5aa6fe32be31, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:27,734 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fd0d0b67d7284fffbc154f0ff28933aa_465c1abe9e468bea9e6a5aa6fe32be31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:27,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742010_1186 (size=12731) 2024-11-15T04:40:27,765 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/9be7caf16a6f4c0c908f3bff8d40b5bc as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9be7caf16a6f4c0c908f3bff8d40b5bc 2024-11-15T04:40:27,773 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/B of 465c1abe9e468bea9e6a5aa6fe32be31 into 9be7caf16a6f4c0c908f3bff8d40b5bc(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:27,773 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:27,773 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/B, priority=13, startTime=1731645627710; duration=0sec 2024-11-15T04:40:27,773 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:27,773 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:B 2024-11-15T04:40:27,773 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:27,775 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:27,775 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/C is initiating minor compaction (all files) 2024-11-15T04:40:27,775 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/C in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,776 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/6e9654ca510d45b1863a07272badd03f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/391fa96d1d8c462688e162d4ebbe1758, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/32c2f33265bd487f9377e06f5929b06e] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=36.1 K 2024-11-15T04:40:27,776 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e9654ca510d45b1863a07272badd03f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731645623127 2024-11-15T04:40:27,777 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 391fa96d1d8c462688e162d4ebbe1758, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731645624364 2024-11-15T04:40:27,777 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 32c2f33265bd487f9377e06f5929b06e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731645625010 2024-11-15T04:40:27,797 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#C#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:27,798 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/e8bc5b4580154a5ab84f3bdbb5fa08da is 50, key is test_row_0/C:col10/1731645625010/Put/seqid=0 2024-11-15T04:40:27,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742011_1187 (size=4469) 2024-11-15T04:40:27,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742012_1188 (size=12731) 2024-11-15T04:40:27,826 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:27,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T04:40:27,831 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/e8bc5b4580154a5ab84f3bdbb5fa08da as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/e8bc5b4580154a5ab84f3bdbb5fa08da 2024-11-15T04:40:27,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115442c648da42c4ae7a2c924bdbec0213c_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645627196/Put/seqid=0 2024-11-15T04:40:27,837 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/C of 465c1abe9e468bea9e6a5aa6fe32be31 into e8bc5b4580154a5ab84f3bdbb5fa08da(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:27,837 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:27,837 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/C, priority=13, startTime=1731645627711; duration=0sec 2024-11-15T04:40:27,837 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:27,837 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:C 2024-11-15T04:40:27,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742013_1189 (size=14994) 2024-11-15T04:40:27,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645687846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645687847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645687847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645687848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645687848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:27,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:27,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T04:40:27,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645687953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645687953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645687953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645687963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645687963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:28,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:28,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:28,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:28,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645688163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645688163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645688163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645688173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645688173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,202 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#A#compaction#154 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:28,203 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/b127d6d4763e4715b991703c9131b5d3 is 175, key is test_row_0/A:col10/1731645625010/Put/seqid=0 2024-11-15T04:40:28,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742014_1190 (size=31685) 2024-11-15T04:40:28,236 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/b127d6d4763e4715b991703c9131b5d3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b127d6d4763e4715b991703c9131b5d3 2024-11-15T04:40:28,242 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/A of 465c1abe9e468bea9e6a5aa6fe32be31 into b127d6d4763e4715b991703c9131b5d3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:28,242 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:28,242 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/A, priority=13, startTime=1731645627710; duration=0sec 2024-11-15T04:40:28,243 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:28,243 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:A 2024-11-15T04:40:28,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:28,253 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115442c648da42c4ae7a2c924bdbec0213c_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115442c648da42c4ae7a2c924bdbec0213c_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:28,254 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/e37ae34a2f944062b2d8e6b9ce13ec93, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:28,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/e37ae34a2f944062b2d8e6b9ce13ec93 is 175, key is test_row_0/A:col10/1731645627196/Put/seqid=0 2024-11-15T04:40:28,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742015_1191 (size=39949) 2024-11-15T04:40:28,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T04:40:28,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645688473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645688473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645688473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645688483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645688483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,669 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/e37ae34a2f944062b2d8e6b9ce13ec93 2024-11-15T04:40:28,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/6a247e99fc4c406c826752e1f8b9c2dc is 50, key is test_row_0/B:col10/1731645627196/Put/seqid=0 2024-11-15T04:40:28,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742016_1192 (size=12301) 2024-11-15T04:40:28,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/6a247e99fc4c406c826752e1f8b9c2dc 2024-11-15T04:40:28,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/2ceaa8424b3840d186c748e8e8b619a5 is 50, key is test_row_0/C:col10/1731645627196/Put/seqid=0 2024-11-15T04:40:28,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742017_1193 (size=12301) 2024-11-15T04:40:28,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:28,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:28,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:28,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:28,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645688982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645688982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645688982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645688993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645688993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:28,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:29,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:29,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:29,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:29,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:29,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:29,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:29,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/2ceaa8424b3840d186c748e8e8b619a5 2024-11-15T04:40:29,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/e37ae34a2f944062b2d8e6b9ce13ec93 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/e37ae34a2f944062b2d8e6b9ce13ec93 2024-11-15T04:40:29,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/e37ae34a2f944062b2d8e6b9ce13ec93, entries=200, sequenceid=277, filesize=39.0 K 2024-11-15T04:40:29,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/6a247e99fc4c406c826752e1f8b9c2dc as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6a247e99fc4c406c826752e1f8b9c2dc 2024-11-15T04:40:29,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6a247e99fc4c406c826752e1f8b9c2dc, entries=150, sequenceid=277, filesize=12.0 K 2024-11-15T04:40:29,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/2ceaa8424b3840d186c748e8e8b619a5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2ceaa8424b3840d186c748e8e8b619a5 2024-11-15T04:40:29,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2ceaa8424b3840d186c748e8e8b619a5, entries=150, sequenceid=277, filesize=12.0 K 2024-11-15T04:40:29,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1390ms, sequenceid=277, compaction requested=false 2024-11-15T04:40:29,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:29,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T04:40:29,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:29,238 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T04:40:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155848f2290a7d41d49732ab7ea6bc2565_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645627831/Put/seqid=0 2024-11-15T04:40:29,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742018_1194 (size=12454) 2024-11-15T04:40:29,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:29,320 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155848f2290a7d41d49732ab7ea6bc2565_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155848f2290a7d41d49732ab7ea6bc2565_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ad2757b30d6a465a9a00648f23ff8050, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:29,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ad2757b30d6a465a9a00648f23ff8050 is 175, key is test_row_0/A:col10/1731645627831/Put/seqid=0 2024-11-15T04:40:29,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742019_1195 (size=31255) 2024-11-15T04:40:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T04:40:29,750 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ad2757b30d6a465a9a00648f23ff8050 2024-11-15T04:40:29,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/8fef2aa92e834425a4c404cc0f927ec1 is 50, key is test_row_0/B:col10/1731645627831/Put/seqid=0 2024-11-15T04:40:29,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742020_1196 (size=12301) 2024-11-15T04:40:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:30,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:30,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645690027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645690026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645690027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645690031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645690032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645690142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645690142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645690143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645690143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645690143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,194 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/8fef2aa92e834425a4c404cc0f927ec1 2024-11-15T04:40:30,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/d454ca6f0d3347f5b23289191a1bc8bf is 50, key is test_row_0/C:col10/1731645627831/Put/seqid=0 2024-11-15T04:40:30,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742021_1197 (size=12301) 2024-11-15T04:40:30,231 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/d454ca6f0d3347f5b23289191a1bc8bf 2024-11-15T04:40:30,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ad2757b30d6a465a9a00648f23ff8050 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ad2757b30d6a465a9a00648f23ff8050 2024-11-15T04:40:30,250 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ad2757b30d6a465a9a00648f23ff8050, entries=150, sequenceid=293, filesize=30.5 K 2024-11-15T04:40:30,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/8fef2aa92e834425a4c404cc0f927ec1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8fef2aa92e834425a4c404cc0f927ec1 2024-11-15T04:40:30,257 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8fef2aa92e834425a4c404cc0f927ec1, entries=150, sequenceid=293, filesize=12.0 K 2024-11-15T04:40:30,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/d454ca6f0d3347f5b23289191a1bc8bf as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d454ca6f0d3347f5b23289191a1bc8bf 2024-11-15T04:40:30,265 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d454ca6f0d3347f5b23289191a1bc8bf, entries=150, sequenceid=293, filesize=12.0 K 2024-11-15T04:40:30,267 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1028ms, sequenceid=293, compaction requested=true 2024-11-15T04:40:30,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:30,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:30,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T04:40:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T04:40:30,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T04:40:30,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0340 sec 2024-11-15T04:40:30,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 3.0410 sec 2024-11-15T04:40:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:30,358 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:30,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T04:40:30,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152bf9bec0e94441ccb08dceca218256d0_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645630357/Put/seqid=0 2024-11-15T04:40:30,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645690365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645690366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645690371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645690372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645690373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742022_1198 (size=14994) 2024-11-15T04:40:30,391 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:30,397 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152bf9bec0e94441ccb08dceca218256d0_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152bf9bec0e94441ccb08dceca218256d0_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:30,399 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/fa84f052e770420ea19e283f9edce6d5, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:30,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/fa84f052e770420ea19e283f9edce6d5 is 175, key is test_row_0/A:col10/1731645630357/Put/seqid=0 2024-11-15T04:40:30,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742023_1199 (size=39949) 2024-11-15T04:40:30,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645690482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645690483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645690483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645690483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645690493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645690693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645690693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645690693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645690703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645690703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:30,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:30,814 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/fa84f052e770420ea19e283f9edce6d5 2024-11-15T04:40:30,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/c0279b1f3fff4676af99e9a54693d076 is 50, key is test_row_0/B:col10/1731645630357/Put/seqid=0 2024-11-15T04:40:30,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742024_1200 (size=12301) 2024-11-15T04:40:30,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/c0279b1f3fff4676af99e9a54693d076 2024-11-15T04:40:30,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/d0d7aecfa9b34f05a36d984ee8e6b151 is 50, key is test_row_0/C:col10/1731645630357/Put/seqid=0 2024-11-15T04:40:30,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742025_1201 (size=12301) 2024-11-15T04:40:30,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/d0d7aecfa9b34f05a36d984ee8e6b151 2024-11-15T04:40:30,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/fa84f052e770420ea19e283f9edce6d5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/fa84f052e770420ea19e283f9edce6d5 2024-11-15T04:40:30,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/fa84f052e770420ea19e283f9edce6d5, entries=200, sequenceid=316, filesize=39.0 K 2024-11-15T04:40:30,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/c0279b1f3fff4676af99e9a54693d076 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c0279b1f3fff4676af99e9a54693d076 2024-11-15T04:40:30,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c0279b1f3fff4676af99e9a54693d076, entries=150, sequenceid=316, filesize=12.0 K 2024-11-15T04:40:30,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/d0d7aecfa9b34f05a36d984ee8e6b151 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d0d7aecfa9b34f05a36d984ee8e6b151 2024-11-15T04:40:30,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d0d7aecfa9b34f05a36d984ee8e6b151, entries=150, sequenceid=316, filesize=12.0 K 2024-11-15T04:40:30,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 465c1abe9e468bea9e6a5aa6fe32be31 in 515ms, sequenceid=316, compaction requested=true 2024-11-15T04:40:30,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:30,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:30,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:30,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:30,873 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:30,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:30,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:30,873 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:30,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:30,876 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:30,876 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/B is initiating minor compaction (all files) 2024-11-15T04:40:30,876 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/B in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:30,876 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9be7caf16a6f4c0c908f3bff8d40b5bc, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6a247e99fc4c406c826752e1f8b9c2dc, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8fef2aa92e834425a4c404cc0f927ec1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c0279b1f3fff4676af99e9a54693d076] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=48.5 K 2024-11-15T04:40:30,876 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be7caf16a6f4c0c908f3bff8d40b5bc, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731645625010 2024-11-15T04:40:30,877 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a247e99fc4c406c826752e1f8b9c2dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731645627196 2024-11-15T04:40:30,878 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142838 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:30,878 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fef2aa92e834425a4c404cc0f927ec1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731645627831 2024-11-15T04:40:30,878 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/A is initiating minor compaction (all files) 2024-11-15T04:40:30,878 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/A in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:30,878 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b127d6d4763e4715b991703c9131b5d3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/e37ae34a2f944062b2d8e6b9ce13ec93, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ad2757b30d6a465a9a00648f23ff8050, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/fa84f052e770420ea19e283f9edce6d5] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=139.5 K 2024-11-15T04:40:30,878 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:30,878 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b127d6d4763e4715b991703c9131b5d3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/e37ae34a2f944062b2d8e6b9ce13ec93, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ad2757b30d6a465a9a00648f23ff8050, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/fa84f052e770420ea19e283f9edce6d5] 2024-11-15T04:40:30,878 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c0279b1f3fff4676af99e9a54693d076, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731645630029 2024-11-15T04:40:30,881 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b127d6d4763e4715b991703c9131b5d3, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731645625010 2024-11-15T04:40:30,882 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting e37ae34a2f944062b2d8e6b9ce13ec93, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731645627193 2024-11-15T04:40:30,882 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad2757b30d6a465a9a00648f23ff8050, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731645627831 2024-11-15T04:40:30,883 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa84f052e770420ea19e283f9edce6d5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731645630024 2024-11-15T04:40:30,901 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:30,901 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/3f30abb804b7419491c9881573294718 is 50, key is test_row_0/B:col10/1731645630357/Put/seqid=0 2024-11-15T04:40:30,911 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:30,942 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115cd6bdb779a5a48c5a38f918c8a062f87_465c1abe9e468bea9e6a5aa6fe32be31 store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:30,944 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115cd6bdb779a5a48c5a38f918c8a062f87_465c1abe9e468bea9e6a5aa6fe32be31, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:30,944 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cd6bdb779a5a48c5a38f918c8a062f87_465c1abe9e468bea9e6a5aa6fe32be31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:30,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742027_1203 (size=4469) 2024-11-15T04:40:30,955 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#A#compaction#166 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:30,956 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/a7b5f3b154594e1a9e54b0d95920b750 is 175, key is test_row_0/A:col10/1731645630357/Put/seqid=0 2024-11-15T04:40:30,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742026_1202 (size=13017) 2024-11-15T04:40:30,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742028_1204 (size=31971) 2024-11-15T04:40:30,970 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/3f30abb804b7419491c9881573294718 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/3f30abb804b7419491c9881573294718 2024-11-15T04:40:30,978 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/B of 465c1abe9e468bea9e6a5aa6fe32be31 into 3f30abb804b7419491c9881573294718(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:30,978 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:30,978 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/B, priority=12, startTime=1731645630873; duration=0sec 2024-11-15T04:40:30,978 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:30,978 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:B 2024-11-15T04:40:30,978 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:30,980 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:30,981 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/C is initiating minor compaction (all files) 2024-11-15T04:40:30,981 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/C in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:30,981 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/e8bc5b4580154a5ab84f3bdbb5fa08da, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2ceaa8424b3840d186c748e8e8b619a5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d454ca6f0d3347f5b23289191a1bc8bf, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d0d7aecfa9b34f05a36d984ee8e6b151] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=48.5 K 2024-11-15T04:40:30,981 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e8bc5b4580154a5ab84f3bdbb5fa08da, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731645625010 2024-11-15T04:40:30,982 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ceaa8424b3840d186c748e8e8b619a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731645627196 2024-11-15T04:40:30,982 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting d454ca6f0d3347f5b23289191a1bc8bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731645627831 2024-11-15T04:40:30,982 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting d0d7aecfa9b34f05a36d984ee8e6b151, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731645630029 2024-11-15T04:40:30,997 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#C#compaction#167 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:30,998 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/714ba4fb5c80491c8a4f059ca27250c3 is 50, key is test_row_0/C:col10/1731645630357/Put/seqid=0 2024-11-15T04:40:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:31,011 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:31,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T04:40:31,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742029_1205 (size=13017) 2024-11-15T04:40:31,037 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/714ba4fb5c80491c8a4f059ca27250c3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/714ba4fb5c80491c8a4f059ca27250c3 2024-11-15T04:40:31,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645691038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645691040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111566c39e620e91460693daa10ad6f451c5_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645630360/Put/seqid=0 2024-11-15T04:40:31,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645691040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,049 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/C of 465c1abe9e468bea9e6a5aa6fe32be31 into 714ba4fb5c80491c8a4f059ca27250c3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:31,049 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:31,049 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/C, priority=12, startTime=1731645630873; duration=0sec 2024-11-15T04:40:31,049 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:31,049 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:C 2024-11-15T04:40:31,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645691044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645691053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742030_1206 (size=14994) 2024-11-15T04:40:31,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:31,074 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111566c39e620e91460693daa10ad6f451c5_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111566c39e620e91460693daa10ad6f451c5_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:31,076 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/7dd14550cd9042dfb9ba3b6a159e6b00, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:31,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/7dd14550cd9042dfb9ba3b6a159e6b00 is 175, key is test_row_0/A:col10/1731645630360/Put/seqid=0 2024-11-15T04:40:31,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742031_1207 (size=39949) 2024-11-15T04:40:31,104 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/7dd14550cd9042dfb9ba3b6a159e6b00 2024-11-15T04:40:31,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/4f8f374c6ca24ca4a7ab67b30586167f is 50, key is test_row_0/B:col10/1731645630360/Put/seqid=0 2024-11-15T04:40:31,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645691153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645691153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645691153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645691163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645691164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742032_1208 (size=12301) 2024-11-15T04:40:31,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/4f8f374c6ca24ca4a7ab67b30586167f 2024-11-15T04:40:31,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/01bd0755da254d92943f799fe0cbf703 is 50, key is test_row_0/C:col10/1731645630360/Put/seqid=0 2024-11-15T04:40:31,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742033_1209 (size=12301) 2024-11-15T04:40:31,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/01bd0755da254d92943f799fe0cbf703 2024-11-15T04:40:31,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/7dd14550cd9042dfb9ba3b6a159e6b00 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7dd14550cd9042dfb9ba3b6a159e6b00 2024-11-15T04:40:31,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7dd14550cd9042dfb9ba3b6a159e6b00, entries=200, sequenceid=333, filesize=39.0 K 2024-11-15T04:40:31,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/4f8f374c6ca24ca4a7ab67b30586167f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4f8f374c6ca24ca4a7ab67b30586167f 2024-11-15T04:40:31,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4f8f374c6ca24ca4a7ab67b30586167f, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T04:40:31,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/01bd0755da254d92943f799fe0cbf703 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/01bd0755da254d92943f799fe0cbf703 2024-11-15T04:40:31,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/01bd0755da254d92943f799fe0cbf703, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T04:40:31,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 465c1abe9e468bea9e6a5aa6fe32be31 in 217ms, sequenceid=333, compaction requested=false 2024-11-15T04:40:31,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:31,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:31,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T04:40:31,382 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/a7b5f3b154594e1a9e54b0d95920b750 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a7b5f3b154594e1a9e54b0d95920b750 2024-11-15T04:40:31,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a382aaab5613462eacdb860198be7135_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645631040/Put/seqid=0 2024-11-15T04:40:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T04:40:31,393 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:31,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:31,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T04:40:31,396 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:31,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:31,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T04:40:31,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742034_1210 (size=14994) 2024-11-15T04:40:31,401 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:31,405 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a382aaab5613462eacdb860198be7135_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a382aaab5613462eacdb860198be7135_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:31,406 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/cfdae775749849628661ece32ff51bd0, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:31,406 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/A of 465c1abe9e468bea9e6a5aa6fe32be31 into a7b5f3b154594e1a9e54b0d95920b750(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:31,406 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:31,406 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/A, priority=12, startTime=1731645630873; duration=0sec 2024-11-15T04:40:31,406 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:31,406 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:A 2024-11-15T04:40:31,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/cfdae775749849628661ece32ff51bd0 is 175, key is test_row_0/A:col10/1731645631040/Put/seqid=0 2024-11-15T04:40:31,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645691420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645691420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645691421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645691421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645691423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742035_1211 (size=39949) 2024-11-15T04:40:31,435 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=355, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/cfdae775749849628661ece32ff51bd0 2024-11-15T04:40:31,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/4a6cf4f8d98643e69811e8ed7d22ad0b is 50, key is test_row_0/B:col10/1731645631040/Put/seqid=0 2024-11-15T04:40:31,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742036_1212 (size=12301) 2024-11-15T04:40:31,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/4a6cf4f8d98643e69811e8ed7d22ad0b 2024-11-15T04:40:31,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/7d34b868989f4441bc220b6355761d1d is 50, key is test_row_0/C:col10/1731645631040/Put/seqid=0 2024-11-15T04:40:31,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742037_1213 (size=12301) 2024-11-15T04:40:31,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/7d34b868989f4441bc220b6355761d1d 2024-11-15T04:40:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T04:40:31,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/cfdae775749849628661ece32ff51bd0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/cfdae775749849628661ece32ff51bd0 2024-11-15T04:40:31,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/cfdae775749849628661ece32ff51bd0, entries=200, sequenceid=355, filesize=39.0 K 2024-11-15T04:40:31,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/4a6cf4f8d98643e69811e8ed7d22ad0b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4a6cf4f8d98643e69811e8ed7d22ad0b 2024-11-15T04:40:31,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4a6cf4f8d98643e69811e8ed7d22ad0b, entries=150, sequenceid=355, filesize=12.0 K 2024-11-15T04:40:31,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/7d34b868989f4441bc220b6355761d1d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7d34b868989f4441bc220b6355761d1d 2024-11-15T04:40:31,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645691533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645691533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645691533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645691533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645691533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7d34b868989f4441bc220b6355761d1d, entries=150, sequenceid=355, filesize=12.0 K 2024-11-15T04:40:31,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 465c1abe9e468bea9e6a5aa6fe32be31 in 180ms, sequenceid=355, compaction requested=true 2024-11-15T04:40:31,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:31,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:31,544 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:31,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:31,544 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:31,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:31,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:31,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:31,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:31,547 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:31,547 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:31,547 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/A is initiating minor compaction (all files) 2024-11-15T04:40:31,547 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/B is initiating minor compaction (all files) 2024-11-15T04:40:31,547 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/B in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:31,547 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/A in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:31,547 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a7b5f3b154594e1a9e54b0d95920b750, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7dd14550cd9042dfb9ba3b6a159e6b00, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/cfdae775749849628661ece32ff51bd0] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=109.2 K 2024-11-15T04:40:31,547 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/3f30abb804b7419491c9881573294718, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4f8f374c6ca24ca4a7ab67b30586167f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4a6cf4f8d98643e69811e8ed7d22ad0b] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=36.7 K 2024-11-15T04:40:31,547 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:31,547 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a7b5f3b154594e1a9e54b0d95920b750, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7dd14550cd9042dfb9ba3b6a159e6b00, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/cfdae775749849628661ece32ff51bd0] 2024-11-15T04:40:31,549 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f30abb804b7419491c9881573294718, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731645630029 2024-11-15T04:40:31,549 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7b5f3b154594e1a9e54b0d95920b750, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731645630029 2024-11-15T04:40:31,549 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dd14550cd9042dfb9ba3b6a159e6b00, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731645630360 2024-11-15T04:40:31,549 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f8f374c6ca24ca4a7ab67b30586167f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731645630360 2024-11-15T04:40:31,550 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a6cf4f8d98643e69811e8ed7d22ad0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731645631031 2024-11-15T04:40:31,550 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfdae775749849628661ece32ff51bd0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731645631031 2024-11-15T04:40:31,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T04:40:31,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:31,552 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T04:40:31,561 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:31,561 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/2c1c2bb2c9934c9bbd42d9469abcc790 is 50, key is test_row_0/B:col10/1731645631040/Put/seqid=0 2024-11-15T04:40:31,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156ea00dad30b44d6b92312908c9463f21_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645631419/Put/seqid=0 2024-11-15T04:40:31,567 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:31,571 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411153e07aefc41d34c20809b9d6a8a1a81eb_465c1abe9e468bea9e6a5aa6fe32be31 store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:31,573 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411153e07aefc41d34c20809b9d6a8a1a81eb_465c1abe9e468bea9e6a5aa6fe32be31, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:31,573 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153e07aefc41d34c20809b9d6a8a1a81eb_465c1abe9e468bea9e6a5aa6fe32be31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:31,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742038_1214 (size=13119) 2024-11-15T04:40:31,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742039_1215 (size=12454) 2024-11-15T04:40:31,590 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/2c1c2bb2c9934c9bbd42d9469abcc790 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/2c1c2bb2c9934c9bbd42d9469abcc790 2024-11-15T04:40:31,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742040_1216 (size=4469) 2024-11-15T04:40:31,598 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/B of 465c1abe9e468bea9e6a5aa6fe32be31 into 2c1c2bb2c9934c9bbd42d9469abcc790(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:31,598 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:31,598 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/B, priority=13, startTime=1731645631544; duration=0sec 2024-11-15T04:40:31,598 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:31,598 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:B 2024-11-15T04:40:31,598 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:31,599 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:31,599 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/C is initiating minor compaction (all files) 2024-11-15T04:40:31,599 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/C in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:31,600 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/714ba4fb5c80491c8a4f059ca27250c3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/01bd0755da254d92943f799fe0cbf703, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7d34b868989f4441bc220b6355761d1d] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=36.7 K 2024-11-15T04:40:31,600 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 714ba4fb5c80491c8a4f059ca27250c3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731645630029 2024-11-15T04:40:31,601 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 01bd0755da254d92943f799fe0cbf703, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731645630360 2024-11-15T04:40:31,602 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d34b868989f4441bc220b6355761d1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731645631031 2024-11-15T04:40:31,612 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:31,613 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/dada6e8006174dc9b6dcc4b4f173252b is 50, key is test_row_0/C:col10/1731645631040/Put/seqid=0 2024-11-15T04:40:31,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742041_1217 (size=13119) 2024-11-15T04:40:31,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T04:40:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:31,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645691768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645691768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645691773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645691774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645691780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645691883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645691883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645691883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645691883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645691892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:31,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:31,993 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156ea00dad30b44d6b92312908c9463f21_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156ea00dad30b44d6b92312908c9463f21_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:31,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ea71b70e77064b009d8defd4c2929a03, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:31,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ea71b70e77064b009d8defd4c2929a03 is 175, key is test_row_0/A:col10/1731645631419/Put/seqid=0 2024-11-15T04:40:31,998 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#A#compaction#176 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:31,999 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/5b10a5860c9947fdb1761e6a10efe13d is 175, key is test_row_0/A:col10/1731645631040/Put/seqid=0 2024-11-15T04:40:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T04:40:32,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742043_1219 (size=32073) 2024-11-15T04:40:32,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742042_1218 (size=31255) 2024-11-15T04:40:32,041 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ea71b70e77064b009d8defd4c2929a03 2024-11-15T04:40:32,043 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/dada6e8006174dc9b6dcc4b4f173252b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/dada6e8006174dc9b6dcc4b4f173252b 2024-11-15T04:40:32,047 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/C of 465c1abe9e468bea9e6a5aa6fe32be31 into dada6e8006174dc9b6dcc4b4f173252b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:32,047 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:32,048 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/C, priority=13, startTime=1731645631545; duration=0sec 2024-11-15T04:40:32,048 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:32,048 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:C 2024-11-15T04:40:32,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/068d153ea6a74f54baa34c4633f0203a is 50, key is test_row_0/B:col10/1731645631419/Put/seqid=0 2024-11-15T04:40:32,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742044_1220 (size=12301) 2024-11-15T04:40:32,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645692093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645692093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645692093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645692093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645692102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645692403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645692403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645692403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645692403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645692412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,431 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/5b10a5860c9947fdb1761e6a10efe13d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/5b10a5860c9947fdb1761e6a10efe13d 2024-11-15T04:40:32,440 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/A of 465c1abe9e468bea9e6a5aa6fe32be31 into 5b10a5860c9947fdb1761e6a10efe13d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:32,440 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:32,440 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/A, priority=13, startTime=1731645631544; duration=0sec 2024-11-15T04:40:32,441 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:32,441 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:A 2024-11-15T04:40:32,465 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/068d153ea6a74f54baa34c4633f0203a 2024-11-15T04:40:32,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/7f456716be7040ed9fdf18e19eaa2d3a is 50, key is test_row_0/C:col10/1731645631419/Put/seqid=0 2024-11-15T04:40:32,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742045_1221 (size=12301) 2024-11-15T04:40:32,484 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/7f456716be7040ed9fdf18e19eaa2d3a 2024-11-15T04:40:32,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/ea71b70e77064b009d8defd4c2929a03 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ea71b70e77064b009d8defd4c2929a03 2024-11-15T04:40:32,497 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ea71b70e77064b009d8defd4c2929a03, entries=150, sequenceid=371, filesize=30.5 K 2024-11-15T04:40:32,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/068d153ea6a74f54baa34c4633f0203a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/068d153ea6a74f54baa34c4633f0203a 2024-11-15T04:40:32,510 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/068d153ea6a74f54baa34c4633f0203a, entries=150, sequenceid=371, filesize=12.0 K 2024-11-15T04:40:32,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-15T04:40:32,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/7f456716be7040ed9fdf18e19eaa2d3a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7f456716be7040ed9fdf18e19eaa2d3a 2024-11-15T04:40:32,526 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7f456716be7040ed9fdf18e19eaa2d3a, entries=150, sequenceid=371, filesize=12.0 K 2024-11-15T04:40:32,527 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 465c1abe9e468bea9e6a5aa6fe32be31 in 975ms, sequenceid=371, compaction requested=false 2024-11-15T04:40:32,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:32,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:32,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T04:40:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T04:40:32,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T04:40:32,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1310 sec 2024-11-15T04:40:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T04:40:32,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1390 sec 2024-11-15T04:40:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:32,915 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:32,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T04:40:32,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a0f70a08fa0b4fdd91bd59306cb69058_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645632913/Put/seqid=0 2024-11-15T04:40:32,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645692927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645692933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645692932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645692936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:32,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645692940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:32,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742046_1222 (size=14994) 2024-11-15T04:40:32,958 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:32,963 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a0f70a08fa0b4fdd91bd59306cb69058_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a0f70a08fa0b4fdd91bd59306cb69058_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:32,965 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f868ffd9d23b4fc4b83df6a1d567f36e, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:32,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f868ffd9d23b4fc4b83df6a1d567f36e is 175, key is test_row_0/A:col10/1731645632913/Put/seqid=0 2024-11-15T04:40:32,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742047_1223 (size=39949) 2024-11-15T04:40:32,974 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f868ffd9d23b4fc4b83df6a1d567f36e 2024-11-15T04:40:32,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/ba12b8ce8d1c4e898e5fe3caa10c0faa is 50, key is test_row_0/B:col10/1731645632913/Put/seqid=0 2024-11-15T04:40:32,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742048_1224 (size=12301) 2024-11-15T04:40:32,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/ba12b8ce8d1c4e898e5fe3caa10c0faa 2024-11-15T04:40:33,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/8ea522b247754bce9aac2e769526e7aa is 50, key is test_row_0/C:col10/1731645632913/Put/seqid=0 2024-11-15T04:40:33,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742049_1225 (size=12301) 2024-11-15T04:40:33,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/8ea522b247754bce9aac2e769526e7aa 2024-11-15T04:40:33,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/f868ffd9d23b4fc4b83df6a1d567f36e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f868ffd9d23b4fc4b83df6a1d567f36e 2024-11-15T04:40:33,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f868ffd9d23b4fc4b83df6a1d567f36e, entries=200, sequenceid=397, filesize=39.0 K 2024-11-15T04:40:33,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/ba12b8ce8d1c4e898e5fe3caa10c0faa as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/ba12b8ce8d1c4e898e5fe3caa10c0faa 2024-11-15T04:40:33,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/ba12b8ce8d1c4e898e5fe3caa10c0faa, entries=150, sequenceid=397, filesize=12.0 K 2024-11-15T04:40:33,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/8ea522b247754bce9aac2e769526e7aa as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8ea522b247754bce9aac2e769526e7aa 2024-11-15T04:40:33,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8ea522b247754bce9aac2e769526e7aa, entries=150, sequenceid=397, filesize=12.0 K 2024-11-15T04:40:33,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 465c1abe9e468bea9e6a5aa6fe32be31 in 121ms, sequenceid=397, compaction requested=true 2024-11-15T04:40:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:33,037 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:33,037 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:33,038 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:33,038 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:33,038 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/B is initiating minor compaction (all files) 2024-11-15T04:40:33,038 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/A is initiating minor compaction (all files) 2024-11-15T04:40:33,038 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/B in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:33,038 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/2c1c2bb2c9934c9bbd42d9469abcc790, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/068d153ea6a74f54baa34c4633f0203a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/ba12b8ce8d1c4e898e5fe3caa10c0faa] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=36.8 K 2024-11-15T04:40:33,038 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/A in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:33,038 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/5b10a5860c9947fdb1761e6a10efe13d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ea71b70e77064b009d8defd4c2929a03, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f868ffd9d23b4fc4b83df6a1d567f36e] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=100.9 K 2024-11-15T04:40:33,038 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:33,039 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/5b10a5860c9947fdb1761e6a10efe13d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ea71b70e77064b009d8defd4c2929a03, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f868ffd9d23b4fc4b83df6a1d567f36e] 2024-11-15T04:40:33,041 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b10a5860c9947fdb1761e6a10efe13d, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731645631031 2024-11-15T04:40:33,041 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c1c2bb2c9934c9bbd42d9469abcc790, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731645631031 2024-11-15T04:40:33,042 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 068d153ea6a74f54baa34c4633f0203a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731645631418 2024-11-15T04:40:33,042 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea71b70e77064b009d8defd4c2929a03, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731645631418 2024-11-15T04:40:33,042 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting ba12b8ce8d1c4e898e5fe3caa10c0faa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731645631764 2024-11-15T04:40:33,043 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting f868ffd9d23b4fc4b83df6a1d567f36e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731645631764 2024-11-15T04:40:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:33,045 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:33,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T04:40:33,050 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:33,059 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:33,059 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411152df411a5c88e4f2489855ca1e9cd2982_465c1abe9e468bea9e6a5aa6fe32be31 store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:33,059 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/58b6c67f1f5b4f40b1df3d664816df15 is 50, key is test_row_0/B:col10/1731645632913/Put/seqid=0 2024-11-15T04:40:33,061 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411152df411a5c88e4f2489855ca1e9cd2982_465c1abe9e468bea9e6a5aa6fe32be31, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:33,061 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152df411a5c88e4f2489855ca1e9cd2982_465c1abe9e468bea9e6a5aa6fe32be31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:33,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156374c3e0960146aa9445fa69074fb918_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645632936/Put/seqid=0 2024-11-15T04:40:33,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742050_1226 (size=13221) 2024-11-15T04:40:33,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742051_1227 (size=14994) 2024-11-15T04:40:33,069 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:33,073 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156374c3e0960146aa9445fa69074fb918_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156374c3e0960146aa9445fa69074fb918_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:33,075 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/9358e888e58349a397b631c80a332041, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:33,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/9358e888e58349a397b631c80a332041 is 175, key is test_row_0/A:col10/1731645632936/Put/seqid=0 2024-11-15T04:40:33,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645693082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645693083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645693084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742052_1228 (size=4469) 2024-11-15T04:40:33,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645693086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742053_1229 (size=39949) 2024-11-15T04:40:33,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645693087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645693193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645693193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645693193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645693203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645693202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645693403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645693403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645693403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645693413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645693413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,478 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/58b6c67f1f5b4f40b1df3d664816df15 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/58b6c67f1f5b4f40b1df3d664816df15 2024-11-15T04:40:33,493 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#A#compaction#183 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:33,493 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/7ee40a0bbe734c49bbed8b0606ef3e10 is 175, key is test_row_0/A:col10/1731645632913/Put/seqid=0 2024-11-15T04:40:33,495 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/B of 465c1abe9e468bea9e6a5aa6fe32be31 into 58b6c67f1f5b4f40b1df3d664816df15(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:33,495 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:33,495 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/B, priority=13, startTime=1731645633037; duration=0sec 2024-11-15T04:40:33,495 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:33,495 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:B 2024-11-15T04:40:33,495 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:33,496 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/9358e888e58349a397b631c80a332041 2024-11-15T04:40:33,499 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:33,499 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/C is initiating minor compaction (all files) 2024-11-15T04:40:33,499 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/C in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:33,501 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/dada6e8006174dc9b6dcc4b4f173252b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7f456716be7040ed9fdf18e19eaa2d3a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8ea522b247754bce9aac2e769526e7aa] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=36.8 K 2024-11-15T04:40:33,502 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting dada6e8006174dc9b6dcc4b4f173252b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731645631031 2024-11-15T04:40:33,504 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f456716be7040ed9fdf18e19eaa2d3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1731645631418 2024-11-15T04:40:33,504 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea522b247754bce9aac2e769526e7aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731645631764 2024-11-15T04:40:33,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/7aeb61d3f97a4b46ab988c64aae9e9a1 is 50, key is test_row_0/B:col10/1731645632936/Put/seqid=0 2024-11-15T04:40:33,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742054_1230 (size=32175) 2024-11-15T04:40:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T04:40:33,543 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:33,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-15T04:40:33,546 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T04:40:33,547 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:33,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:33,549 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#C#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:33,550 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/591cfd1c471242899cbd2c57eb9950ec is 50, key is test_row_0/C:col10/1731645632913/Put/seqid=0 2024-11-15T04:40:33,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742055_1231 (size=12301) 2024-11-15T04:40:33,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742056_1232 (size=13221) 2024-11-15T04:40:33,586 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/591cfd1c471242899cbd2c57eb9950ec as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/591cfd1c471242899cbd2c57eb9950ec 2024-11-15T04:40:33,600 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/C of 465c1abe9e468bea9e6a5aa6fe32be31 into 591cfd1c471242899cbd2c57eb9950ec(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:33,600 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:33,601 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/C, priority=13, startTime=1731645633037; duration=0sec 2024-11-15T04:40:33,601 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:33,601 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:C 2024-11-15T04:40:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T04:40:33,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T04:40:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:33,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:33,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:33,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:33,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:33,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645693713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645693713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645693722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645693722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645693723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:33,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:33,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T04:40:33,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:33,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:33,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:33,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:33,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T04:40:33,947 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/7ee40a0bbe734c49bbed8b0606ef3e10 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7ee40a0bbe734c49bbed8b0606ef3e10 2024-11-15T04:40:33,955 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/A of 465c1abe9e468bea9e6a5aa6fe32be31 into 7ee40a0bbe734c49bbed8b0606ef3e10(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:33,955 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:33,955 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/A, priority=13, startTime=1731645633037; duration=0sec 2024-11-15T04:40:33,956 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:33,956 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:A 2024-11-15T04:40:33,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/7aeb61d3f97a4b46ab988c64aae9e9a1 2024-11-15T04:40:33,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/eed41972e9bd48ca9362db1c815a9040 is 50, key is test_row_0/C:col10/1731645632936/Put/seqid=0 2024-11-15T04:40:33,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742057_1233 (size=12301) 2024-11-15T04:40:34,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T04:40:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:34,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:34,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:34,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T04:40:34,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:34,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:34,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:34,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:34,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:34,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T04:40:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645694233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645694233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645694233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645694233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:34,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645694233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:34,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T04:40:34,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:34,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:34,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:34,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:34,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:34,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:34,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/eed41972e9bd48ca9362db1c815a9040 2024-11-15T04:40:34,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/9358e888e58349a397b631c80a332041 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9358e888e58349a397b631c80a332041 2024-11-15T04:40:34,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9358e888e58349a397b631c80a332041, entries=200, sequenceid=411, filesize=39.0 K 2024-11-15T04:40:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/7aeb61d3f97a4b46ab988c64aae9e9a1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/7aeb61d3f97a4b46ab988c64aae9e9a1 2024-11-15T04:40:34,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/7aeb61d3f97a4b46ab988c64aae9e9a1, entries=150, sequenceid=411, filesize=12.0 K 2024-11-15T04:40:34,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/eed41972e9bd48ca9362db1c815a9040 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/eed41972e9bd48ca9362db1c815a9040 2024-11-15T04:40:34,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/eed41972e9bd48ca9362db1c815a9040, entries=150, sequenceid=411, filesize=12.0 K 2024-11-15T04:40:34,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1387ms, sequenceid=411, compaction requested=false 2024-11-15T04:40:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:34,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T04:40:34,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:34,472 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T04:40:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153a1910a16a2c44f395adaf91ec5d79c3_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645633085/Put/seqid=0 2024-11-15T04:40:34,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742058_1234 (size=12454) 2024-11-15T04:40:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:34,524 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153a1910a16a2c44f395adaf91ec5d79c3_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a1910a16a2c44f395adaf91ec5d79c3_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/a8711ae3bbca4dd4ae4f139194325df2, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/a8711ae3bbca4dd4ae4f139194325df2 is 175, key is test_row_0/A:col10/1731645633085/Put/seqid=0 2024-11-15T04:40:34,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742059_1235 (size=31255) 2024-11-15T04:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T04:40:34,967 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=437, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/a8711ae3bbca4dd4ae4f139194325df2 2024-11-15T04:40:34,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/8e9c559adade4272bf27b02fa54d9ae2 is 50, key is test_row_0/B:col10/1731645633085/Put/seqid=0 2024-11-15T04:40:35,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742060_1236 (size=12301) 2024-11-15T04:40:35,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. as already flushing 2024-11-15T04:40:35,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:35,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645695255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645695256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645695257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645695258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645695260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645695363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645695363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645695363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645695363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645695372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,403 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/8e9c559adade4272bf27b02fa54d9ae2 2024-11-15T04:40:35,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/fdd7b7cb72a54001b3c75ad3f6ad620d is 50, key is test_row_0/C:col10/1731645633085/Put/seqid=0 2024-11-15T04:40:35,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742061_1237 (size=12301) 2024-11-15T04:40:35,505 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-15T04:40:35,505 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:35,506 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,506 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,506 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:35,510 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-15T04:40:35,510 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:35,510 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,510 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:35,510 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-11-15T04:40:35,510 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:35,510 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,510 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:35,512 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-11-15T04:40:35,512 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:35,512 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,512 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,512 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:35,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43412 deadline: 1731645695573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43404 deadline: 1731645695573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43390 deadline: 1731645695573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43372 deadline: 1731645695573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43422 deadline: 1731645695583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:35,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:35,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:40:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T04:40:35,830 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/fdd7b7cb72a54001b3c75ad3f6ad620d 2024-11-15T04:40:35,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/a8711ae3bbca4dd4ae4f139194325df2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a8711ae3bbca4dd4ae4f139194325df2 2024-11-15T04:40:35,842 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a8711ae3bbca4dd4ae4f139194325df2, entries=150, sequenceid=437, filesize=30.5 K 2024-11-15T04:40:35,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/8e9c559adade4272bf27b02fa54d9ae2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8e9c559adade4272bf27b02fa54d9ae2 2024-11-15T04:40:35,848 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8e9c559adade4272bf27b02fa54d9ae2, entries=150, sequenceid=437, filesize=12.0 K 2024-11-15T04:40:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/fdd7b7cb72a54001b3c75ad3f6ad620d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/fdd7b7cb72a54001b3c75ad3f6ad620d 2024-11-15T04:40:35,854 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/fdd7b7cb72a54001b3c75ad3f6ad620d, entries=150, sequenceid=437, filesize=12.0 K 2024-11-15T04:40:35,855 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1383ms, sequenceid=437, compaction requested=true 2024-11-15T04:40:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:35,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-15T04:40:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-15T04:40:35,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T04:40:35,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3090 sec 2024-11-15T04:40:35,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.3160 sec 2024-11-15T04:40:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:35,884 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:35,885 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-15T04:40:35,885 INFO [Thread-608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-608. 2024-11-15T04:40:35,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T04:40:35,885 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:35,885 DEBUG [Thread-608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:35,885 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,885 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,885 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-15T04:40:35,885 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:35,885 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,885 DEBUG [Thread-608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,885 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:35,885 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:35,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:35,887 INFO [Thread-606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-606. 2024-11-15T04:40:35,887 DEBUG [Thread-606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:35,887 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,887 DEBUG [Thread-606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:35,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153eb107d493614928859c254d9517198b_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_0/A:col10/1731645635883/Put/seqid=0 2024-11-15T04:40:35,895 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-11-15T04:40:35,895 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:40:35,895 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,895 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:35,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:35,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742062_1238 (size=12454) 2024-11-15T04:40:35,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:35,907 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153eb107d493614928859c254d9517198b_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153eb107d493614928859c254d9517198b_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:35,908 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/091c5e56c1a04f06bd38b47fce413537, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:35,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/091c5e56c1a04f06bd38b47fce413537 is 175, key is test_row_0/A:col10/1731645635883/Put/seqid=0 2024-11-15T04:40:35,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742063_1239 (size=31255) 2024-11-15T04:40:36,313 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=454, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/091c5e56c1a04f06bd38b47fce413537 2024-11-15T04:40:36,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/95ca64ce74104b20859f87321ba18d33 is 50, key is test_row_0/B:col10/1731645635883/Put/seqid=0 2024-11-15T04:40:36,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742064_1240 (size=12301) 2024-11-15T04:40:36,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/95ca64ce74104b20859f87321ba18d33 2024-11-15T04:40:36,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/5fbde6523ab3421cb937eb6b6a74ec5c is 50, key is test_row_0/C:col10/1731645635883/Put/seqid=0 2024-11-15T04:40:36,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742065_1241 (size=12301) 2024-11-15T04:40:36,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/5fbde6523ab3421cb937eb6b6a74ec5c 2024-11-15T04:40:36,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/091c5e56c1a04f06bd38b47fce413537 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/091c5e56c1a04f06bd38b47fce413537 2024-11-15T04:40:36,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/091c5e56c1a04f06bd38b47fce413537, entries=150, sequenceid=454, filesize=30.5 K 2024-11-15T04:40:36,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/95ca64ce74104b20859f87321ba18d33 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/95ca64ce74104b20859f87321ba18d33 2024-11-15T04:40:36,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/95ca64ce74104b20859f87321ba18d33, entries=150, sequenceid=454, filesize=12.0 K 2024-11-15T04:40:36,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/5fbde6523ab3421cb937eb6b6a74ec5c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5fbde6523ab3421cb937eb6b6a74ec5c 2024-11-15T04:40:36,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5fbde6523ab3421cb937eb6b6a74ec5c, entries=150, sequenceid=454, filesize=12.0 K 2024-11-15T04:40:36,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=6.71 KB/6870 for 465c1abe9e468bea9e6a5aa6fe32be31 in 876ms, sequenceid=454, compaction requested=true 2024-11-15T04:40:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:36,760 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 465c1abe9e468bea9e6a5aa6fe32be31:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:36,760 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:36,761 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:36,761 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/B is initiating minor compaction (all files) 2024-11-15T04:40:36,761 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:36,761 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/B in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:36,761 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/A is initiating minor compaction (all files) 2024-11-15T04:40:36,761 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/A in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:36,761 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/58b6c67f1f5b4f40b1df3d664816df15, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/7aeb61d3f97a4b46ab988c64aae9e9a1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8e9c559adade4272bf27b02fa54d9ae2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/95ca64ce74104b20859f87321ba18d33] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=48.9 K 2024-11-15T04:40:36,761 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7ee40a0bbe734c49bbed8b0606ef3e10, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9358e888e58349a397b631c80a332041, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a8711ae3bbca4dd4ae4f139194325df2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/091c5e56c1a04f06bd38b47fce413537] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=131.5 K 2024-11-15T04:40:36,761 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:36,761 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7ee40a0bbe734c49bbed8b0606ef3e10, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9358e888e58349a397b631c80a332041, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a8711ae3bbca4dd4ae4f139194325df2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/091c5e56c1a04f06bd38b47fce413537] 2024-11-15T04:40:36,761 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 58b6c67f1f5b4f40b1df3d664816df15, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731645631764 2024-11-15T04:40:36,762 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aeb61d3f97a4b46ab988c64aae9e9a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731645632931 2024-11-15T04:40:36,762 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ee40a0bbe734c49bbed8b0606ef3e10, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731645631764 2024-11-15T04:40:36,762 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e9c559adade4272bf27b02fa54d9ae2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731645633080 2024-11-15T04:40:36,762 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9358e888e58349a397b631c80a332041, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731645632925 2024-11-15T04:40:36,762 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 95ca64ce74104b20859f87321ba18d33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731645635256 2024-11-15T04:40:36,762 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8711ae3bbca4dd4ae4f139194325df2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731645633080 2024-11-15T04:40:36,763 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 091c5e56c1a04f06bd38b47fce413537, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731645635256 2024-11-15T04:40:36,771 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:36,772 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/37b61ea6d87749939cac61d1878de89a is 50, key is test_row_0/B:col10/1731645635883/Put/seqid=0 2024-11-15T04:40:36,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742066_1242 (size=13357) 2024-11-15T04:40:36,778 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:36,783 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115777ebf48e73b4c52887052e5a7489b32_465c1abe9e468bea9e6a5aa6fe32be31 store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:36,807 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115777ebf48e73b4c52887052e5a7489b32_465c1abe9e468bea9e6a5aa6fe32be31, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:36,808 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115777ebf48e73b4c52887052e5a7489b32_465c1abe9e468bea9e6a5aa6fe32be31 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:36,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742067_1243 (size=4469) 2024-11-15T04:40:37,180 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/37b61ea6d87749939cac61d1878de89a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/37b61ea6d87749939cac61d1878de89a 2024-11-15T04:40:37,187 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/B of 465c1abe9e468bea9e6a5aa6fe32be31 into 37b61ea6d87749939cac61d1878de89a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:37,187 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:37,187 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/B, priority=12, startTime=1731645636760; duration=0sec 2024-11-15T04:40:37,188 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:37,188 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:B 2024-11-15T04:40:37,188 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:37,189 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:37,189 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 465c1abe9e468bea9e6a5aa6fe32be31/C is initiating minor compaction (all files) 2024-11-15T04:40:37,189 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 465c1abe9e468bea9e6a5aa6fe32be31/C in TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:37,190 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/591cfd1c471242899cbd2c57eb9950ec, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/eed41972e9bd48ca9362db1c815a9040, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/fdd7b7cb72a54001b3c75ad3f6ad620d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5fbde6523ab3421cb937eb6b6a74ec5c] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp, totalSize=48.9 K 2024-11-15T04:40:37,190 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 591cfd1c471242899cbd2c57eb9950ec, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731645631764 2024-11-15T04:40:37,190 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting eed41972e9bd48ca9362db1c815a9040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731645632931 2024-11-15T04:40:37,191 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting fdd7b7cb72a54001b3c75ad3f6ad620d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731645633080 2024-11-15T04:40:37,191 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fbde6523ab3421cb937eb6b6a74ec5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731645635256 2024-11-15T04:40:37,202 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:37,202 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/447b60a7228449a9be08641e887058ce is 50, key is test_row_0/C:col10/1731645635883/Put/seqid=0 2024-11-15T04:40:37,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742068_1244 (size=13357) 2024-11-15T04:40:37,215 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 465c1abe9e468bea9e6a5aa6fe32be31#A#compaction#196 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:37,216 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/697de655ec3842b6ba8f38fd6624ab22 is 175, key is test_row_0/A:col10/1731645635883/Put/seqid=0 2024-11-15T04:40:37,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742069_1245 (size=32311) 2024-11-15T04:40:37,611 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/447b60a7228449a9be08641e887058ce as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/447b60a7228449a9be08641e887058ce 2024-11-15T04:40:37,617 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/C of 465c1abe9e468bea9e6a5aa6fe32be31 into 447b60a7228449a9be08641e887058ce(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:37,617 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:37,617 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/C, priority=12, startTime=1731645636760; duration=0sec 2024-11-15T04:40:37,617 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:37,617 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:C 2024-11-15T04:40:37,626 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/697de655ec3842b6ba8f38fd6624ab22 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/697de655ec3842b6ba8f38fd6624ab22 2024-11-15T04:40:37,632 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 465c1abe9e468bea9e6a5aa6fe32be31/A of 465c1abe9e468bea9e6a5aa6fe32be31 into 697de655ec3842b6ba8f38fd6624ab22(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:37,632 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 465c1abe9e468bea9e6a5aa6fe32be31: 2024-11-15T04:40:37,632 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., storeName=465c1abe9e468bea9e6a5aa6fe32be31/A, priority=12, startTime=1731645636760; duration=0sec 2024-11-15T04:40:37,632 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:37,632 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 465c1abe9e468bea9e6a5aa6fe32be31:A 2024-11-15T04:40:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T04:40:37,713 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:37,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T04:40:37,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-15T04:40:37,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-15T04:40:37,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-15T04:40:37,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-15T04:40:37,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-15T04:40:37,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T04:40:37,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3219 2024-11-15T04:40:37,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3150 2024-11-15T04:40:37,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T04:40:37,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1423 2024-11-15T04:40:37,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4269 rows 2024-11-15T04:40:37,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1415 2024-11-15T04:40:37,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4241 rows 2024-11-15T04:40:37,714 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T04:40:37,714 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T04:40:37,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:37,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:37,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:40:37,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T04:40:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T04:40:37,719 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645637718"}]},"ts":"1731645637718"} 2024-11-15T04:40:37,720 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T04:40:37,720 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T04:40:37,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T04:40:37,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, UNASSIGN}] 2024-11-15T04:40:37,724 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, UNASSIGN 2024-11-15T04:40:37,724 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=465c1abe9e468bea9e6a5aa6fe32be31, regionState=CLOSING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:37,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, UNASSIGN because future has completed 2024-11-15T04:40:37,726 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T04:40:37,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:40:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T04:40:37,879 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:37,879 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T04:40:37,879 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 465c1abe9e468bea9e6a5aa6fe32be31, disabling compactions & flushes 2024-11-15T04:40:37,879 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:37,879 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:37,879 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. after waiting 0 ms 2024-11-15T04:40:37,879 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:37,879 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 465c1abe9e468bea9e6a5aa6fe32be31 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-15T04:40:37,885 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158b000b4a245b4b209e8072d51f4bc044_465c1abe9e468bea9e6a5aa6fe32be31 is 50, key is test_row_1/A:col10/1731645635893/Put/seqid=0 2024-11-15T04:40:37,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742070_1246 (size=7374) 2024-11-15T04:40:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T04:40:38,289 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:38,294 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158b000b4a245b4b209e8072d51f4bc044_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158b000b4a245b4b209e8072d51f4bc044_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:38,295 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/27a8716b70a840cd83519b73bc9a8b3c, store: [table=TestAcidGuarantees family=A region=465c1abe9e468bea9e6a5aa6fe32be31] 2024-11-15T04:40:38,296 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/27a8716b70a840cd83519b73bc9a8b3c is 175, key is test_row_1/A:col10/1731645635893/Put/seqid=0 2024-11-15T04:40:38,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742071_1247 (size=13865) 2024-11-15T04:40:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T04:40:38,700 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=461, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/27a8716b70a840cd83519b73bc9a8b3c 2024-11-15T04:40:38,707 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/cc32f44df1f64fec9742b101408b67f2 is 50, key is test_row_1/B:col10/1731645635893/Put/seqid=0 2024-11-15T04:40:38,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742072_1248 (size=7415) 2024-11-15T04:40:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T04:40:39,112 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/cc32f44df1f64fec9742b101408b67f2 2024-11-15T04:40:39,124 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/6a90cb198bc34fcbbb00d1fa37153d04 is 50, key is test_row_1/C:col10/1731645635893/Put/seqid=0 2024-11-15T04:40:39,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742073_1249 (size=7415) 2024-11-15T04:40:39,360 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T04:40:39,529 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/6a90cb198bc34fcbbb00d1fa37153d04 2024-11-15T04:40:39,534 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/A/27a8716b70a840cd83519b73bc9a8b3c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/27a8716b70a840cd83519b73bc9a8b3c 2024-11-15T04:40:39,539 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/27a8716b70a840cd83519b73bc9a8b3c, entries=50, sequenceid=461, filesize=13.5 K 2024-11-15T04:40:39,540 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/B/cc32f44df1f64fec9742b101408b67f2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/cc32f44df1f64fec9742b101408b67f2 2024-11-15T04:40:39,545 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/cc32f44df1f64fec9742b101408b67f2, entries=50, sequenceid=461, filesize=7.2 K 2024-11-15T04:40:39,546 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/.tmp/C/6a90cb198bc34fcbbb00d1fa37153d04 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/6a90cb198bc34fcbbb00d1fa37153d04 2024-11-15T04:40:39,551 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/6a90cb198bc34fcbbb00d1fa37153d04, entries=50, sequenceid=461, filesize=7.2 K 2024-11-15T04:40:39,552 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1673ms, sequenceid=461, compaction requested=false 2024-11-15T04:40:39,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b091d84c6cac497a941aba0a31016212, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/507d7fb728374570bd0f50ffc95f166a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/1c2411b33f274a5b812bfd7586b6c41d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/26ca97a4adc84a9e8be873380a6c7851, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9bafd5140adb4ad58ea5bcafdc22700a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/57f95b0b344941feb1656f9a28f0610d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f14f615b0bbe41a181195396d8e12c5e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/6b723d7f763f44728000ddbdc74332df, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/04418f1e05504e788ecf5c0535517df9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ede1538d7c94496aa6f7e319795333ce, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f107f12288af40be9b2fa9fa7dbd4e5e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/35bc7e44dcd9480cb4fb6415a8b424f1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/2e89413a19e2421096bb3f4c12fccbb5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/dbd34cea6bbb469d92bf3347d726ad76, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/61abc808739e4cfca587bcf25b007d73, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/0f7d80838c1a4e32a7867716a060d897, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/8270e2c29b054eb1ad65e4e590fabb61, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b127d6d4763e4715b991703c9131b5d3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/e37ae34a2f944062b2d8e6b9ce13ec93, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ad2757b30d6a465a9a00648f23ff8050, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/fa84f052e770420ea19e283f9edce6d5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a7b5f3b154594e1a9e54b0d95920b750, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7dd14550cd9042dfb9ba3b6a159e6b00, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/cfdae775749849628661ece32ff51bd0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/5b10a5860c9947fdb1761e6a10efe13d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ea71b70e77064b009d8defd4c2929a03, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f868ffd9d23b4fc4b83df6a1d567f36e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7ee40a0bbe734c49bbed8b0606ef3e10, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9358e888e58349a397b631c80a332041, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a8711ae3bbca4dd4ae4f139194325df2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/091c5e56c1a04f06bd38b47fce413537] to archive 2024-11-15T04:40:39,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:40:39,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b091d84c6cac497a941aba0a31016212 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b091d84c6cac497a941aba0a31016212 2024-11-15T04:40:39,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/507d7fb728374570bd0f50ffc95f166a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/507d7fb728374570bd0f50ffc95f166a 2024-11-15T04:40:39,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/1c2411b33f274a5b812bfd7586b6c41d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/1c2411b33f274a5b812bfd7586b6c41d 2024-11-15T04:40:39,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/26ca97a4adc84a9e8be873380a6c7851 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/26ca97a4adc84a9e8be873380a6c7851 2024-11-15T04:40:39,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9bafd5140adb4ad58ea5bcafdc22700a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9bafd5140adb4ad58ea5bcafdc22700a 2024-11-15T04:40:39,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/57f95b0b344941feb1656f9a28f0610d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/57f95b0b344941feb1656f9a28f0610d 2024-11-15T04:40:39,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f14f615b0bbe41a181195396d8e12c5e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f14f615b0bbe41a181195396d8e12c5e 2024-11-15T04:40:39,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/6b723d7f763f44728000ddbdc74332df to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/6b723d7f763f44728000ddbdc74332df 2024-11-15T04:40:39,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/04418f1e05504e788ecf5c0535517df9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/04418f1e05504e788ecf5c0535517df9 2024-11-15T04:40:39,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ede1538d7c94496aa6f7e319795333ce to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ede1538d7c94496aa6f7e319795333ce 2024-11-15T04:40:39,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f107f12288af40be9b2fa9fa7dbd4e5e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f107f12288af40be9b2fa9fa7dbd4e5e 2024-11-15T04:40:39,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/35bc7e44dcd9480cb4fb6415a8b424f1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/35bc7e44dcd9480cb4fb6415a8b424f1 2024-11-15T04:40:39,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/2e89413a19e2421096bb3f4c12fccbb5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/2e89413a19e2421096bb3f4c12fccbb5 2024-11-15T04:40:39,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/dbd34cea6bbb469d92bf3347d726ad76 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/dbd34cea6bbb469d92bf3347d726ad76 2024-11-15T04:40:39,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/61abc808739e4cfca587bcf25b007d73 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/61abc808739e4cfca587bcf25b007d73 2024-11-15T04:40:39,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/0f7d80838c1a4e32a7867716a060d897 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/0f7d80838c1a4e32a7867716a060d897 2024-11-15T04:40:39,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/8270e2c29b054eb1ad65e4e590fabb61 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/8270e2c29b054eb1ad65e4e590fabb61 2024-11-15T04:40:39,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b127d6d4763e4715b991703c9131b5d3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/b127d6d4763e4715b991703c9131b5d3 2024-11-15T04:40:39,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/e37ae34a2f944062b2d8e6b9ce13ec93 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/e37ae34a2f944062b2d8e6b9ce13ec93 2024-11-15T04:40:39,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ad2757b30d6a465a9a00648f23ff8050 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ad2757b30d6a465a9a00648f23ff8050 2024-11-15T04:40:39,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/fa84f052e770420ea19e283f9edce6d5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/fa84f052e770420ea19e283f9edce6d5 2024-11-15T04:40:39,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a7b5f3b154594e1a9e54b0d95920b750 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a7b5f3b154594e1a9e54b0d95920b750 2024-11-15T04:40:39,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7dd14550cd9042dfb9ba3b6a159e6b00 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7dd14550cd9042dfb9ba3b6a159e6b00 2024-11-15T04:40:39,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/cfdae775749849628661ece32ff51bd0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/cfdae775749849628661ece32ff51bd0 2024-11-15T04:40:39,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/5b10a5860c9947fdb1761e6a10efe13d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/5b10a5860c9947fdb1761e6a10efe13d 2024-11-15T04:40:39,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ea71b70e77064b009d8defd4c2929a03 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/ea71b70e77064b009d8defd4c2929a03 2024-11-15T04:40:39,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f868ffd9d23b4fc4b83df6a1d567f36e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/f868ffd9d23b4fc4b83df6a1d567f36e 2024-11-15T04:40:39,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7ee40a0bbe734c49bbed8b0606ef3e10 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/7ee40a0bbe734c49bbed8b0606ef3e10 2024-11-15T04:40:39,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9358e888e58349a397b631c80a332041 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/9358e888e58349a397b631c80a332041 2024-11-15T04:40:39,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a8711ae3bbca4dd4ae4f139194325df2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/a8711ae3bbca4dd4ae4f139194325df2 2024-11-15T04:40:39,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/091c5e56c1a04f06bd38b47fce413537 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/091c5e56c1a04f06bd38b47fce413537 2024-11-15T04:40:39,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c654b87037b6492c86dd4e79e13d75e7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e41dd12cbfc149a483289831d4a57dd9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/1beba43a83d740d29f51cdd290b56b42, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/651107f6f0e84d2cb4b828b4836b5652, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6dc67441e0e74ec494fdd778fb807685, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/df313646453f4248a4f6ce75a3913bca, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/092fb273d6ed4730b26d957099e12af4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/bc88b36bdf6e49a6b21b5c046bac03ee, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/56a4f8927dc943a2ad0de18c9e3e02c9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e9b05e0f96814931bba382c0263179ab, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/fcda5f8773cf4027beeeaaf31281294a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/15a04b33486142e095d34e177071a724, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/de53ad56b7954f1086a085126a1911c2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c6fe827cd2ad435184a6c37033dc2f78, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6d4b936087624f868d5c356c840d631c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/d94bfaa4bd764fecba638c2d1a7caa95, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9be7caf16a6f4c0c908f3bff8d40b5bc, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9ce1cf7a37dc4c1c89a95a68a277d8f3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6a247e99fc4c406c826752e1f8b9c2dc, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8fef2aa92e834425a4c404cc0f927ec1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/3f30abb804b7419491c9881573294718, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c0279b1f3fff4676af99e9a54693d076, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4f8f374c6ca24ca4a7ab67b30586167f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/2c1c2bb2c9934c9bbd42d9469abcc790, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4a6cf4f8d98643e69811e8ed7d22ad0b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/068d153ea6a74f54baa34c4633f0203a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/58b6c67f1f5b4f40b1df3d664816df15, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/ba12b8ce8d1c4e898e5fe3caa10c0faa, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/7aeb61d3f97a4b46ab988c64aae9e9a1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8e9c559adade4272bf27b02fa54d9ae2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/95ca64ce74104b20859f87321ba18d33] to archive 2024-11-15T04:40:39,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:40:39,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c654b87037b6492c86dd4e79e13d75e7 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c654b87037b6492c86dd4e79e13d75e7 2024-11-15T04:40:39,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e41dd12cbfc149a483289831d4a57dd9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e41dd12cbfc149a483289831d4a57dd9 2024-11-15T04:40:39,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/1beba43a83d740d29f51cdd290b56b42 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/1beba43a83d740d29f51cdd290b56b42 2024-11-15T04:40:39,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/651107f6f0e84d2cb4b828b4836b5652 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/651107f6f0e84d2cb4b828b4836b5652 2024-11-15T04:40:39,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6dc67441e0e74ec494fdd778fb807685 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6dc67441e0e74ec494fdd778fb807685 2024-11-15T04:40:39,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/df313646453f4248a4f6ce75a3913bca to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/df313646453f4248a4f6ce75a3913bca 2024-11-15T04:40:39,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/092fb273d6ed4730b26d957099e12af4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/092fb273d6ed4730b26d957099e12af4 2024-11-15T04:40:39,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/bc88b36bdf6e49a6b21b5c046bac03ee to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/bc88b36bdf6e49a6b21b5c046bac03ee 2024-11-15T04:40:39,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/56a4f8927dc943a2ad0de18c9e3e02c9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/56a4f8927dc943a2ad0de18c9e3e02c9 2024-11-15T04:40:39,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e9b05e0f96814931bba382c0263179ab to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/e9b05e0f96814931bba382c0263179ab 2024-11-15T04:40:39,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/fcda5f8773cf4027beeeaaf31281294a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/fcda5f8773cf4027beeeaaf31281294a 2024-11-15T04:40:39,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/15a04b33486142e095d34e177071a724 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/15a04b33486142e095d34e177071a724 2024-11-15T04:40:39,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/de53ad56b7954f1086a085126a1911c2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/de53ad56b7954f1086a085126a1911c2 2024-11-15T04:40:39,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c6fe827cd2ad435184a6c37033dc2f78 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c6fe827cd2ad435184a6c37033dc2f78 2024-11-15T04:40:39,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6d4b936087624f868d5c356c840d631c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6d4b936087624f868d5c356c840d631c 2024-11-15T04:40:39,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/d94bfaa4bd764fecba638c2d1a7caa95 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/d94bfaa4bd764fecba638c2d1a7caa95 2024-11-15T04:40:39,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9be7caf16a6f4c0c908f3bff8d40b5bc to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9be7caf16a6f4c0c908f3bff8d40b5bc 2024-11-15T04:40:39,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9ce1cf7a37dc4c1c89a95a68a277d8f3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/9ce1cf7a37dc4c1c89a95a68a277d8f3 2024-11-15T04:40:39,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6a247e99fc4c406c826752e1f8b9c2dc to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/6a247e99fc4c406c826752e1f8b9c2dc 2024-11-15T04:40:39,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8fef2aa92e834425a4c404cc0f927ec1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8fef2aa92e834425a4c404cc0f927ec1 2024-11-15T04:40:39,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/3f30abb804b7419491c9881573294718 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/3f30abb804b7419491c9881573294718 2024-11-15T04:40:39,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c0279b1f3fff4676af99e9a54693d076 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/c0279b1f3fff4676af99e9a54693d076 2024-11-15T04:40:39,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4f8f374c6ca24ca4a7ab67b30586167f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4f8f374c6ca24ca4a7ab67b30586167f 2024-11-15T04:40:39,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/2c1c2bb2c9934c9bbd42d9469abcc790 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/2c1c2bb2c9934c9bbd42d9469abcc790 2024-11-15T04:40:39,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4a6cf4f8d98643e69811e8ed7d22ad0b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/4a6cf4f8d98643e69811e8ed7d22ad0b 2024-11-15T04:40:39,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/068d153ea6a74f54baa34c4633f0203a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/068d153ea6a74f54baa34c4633f0203a 2024-11-15T04:40:39,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/58b6c67f1f5b4f40b1df3d664816df15 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/58b6c67f1f5b4f40b1df3d664816df15 2024-11-15T04:40:39,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/ba12b8ce8d1c4e898e5fe3caa10c0faa to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/ba12b8ce8d1c4e898e5fe3caa10c0faa 2024-11-15T04:40:39,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/7aeb61d3f97a4b46ab988c64aae9e9a1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/7aeb61d3f97a4b46ab988c64aae9e9a1 2024-11-15T04:40:39,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8e9c559adade4272bf27b02fa54d9ae2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/8e9c559adade4272bf27b02fa54d9ae2 2024-11-15T04:40:39,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/95ca64ce74104b20859f87321ba18d33 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/95ca64ce74104b20859f87321ba18d33 2024-11-15T04:40:39,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/20afd1e7e2dd4908a369dc8734635702, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9689c61b42c94973964391a37edfb40c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2deaacbcb22c458b82dc8b9bb59c9184, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8beb2789990c42cf8a8ad529b31e25c3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ce09d5b6ae714b0d9a83b5981d05efb1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5ca62f0085574f7cb4690af1a7fb25a3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/f457e88170db4c5ba769894c1a01ce65, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/081ef298f95d4b18a30b99ab1a909695, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/c74f4c3978134a5383932829e44aad1b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9b4668b2bd7345ab9e99f4b1c298a19e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ddbbf08e9c404d2a966d38cc1001d566, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/229326c7741041d59851182ca2e1885b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/36a146b810b8494f80ce58859749831d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/6e9654ca510d45b1863a07272badd03f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/438d5b28b7c24358a202b3111510c5ed, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/391fa96d1d8c462688e162d4ebbe1758, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/e8bc5b4580154a5ab84f3bdbb5fa08da, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/32c2f33265bd487f9377e06f5929b06e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2ceaa8424b3840d186c748e8e8b619a5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d454ca6f0d3347f5b23289191a1bc8bf, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/714ba4fb5c80491c8a4f059ca27250c3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d0d7aecfa9b34f05a36d984ee8e6b151, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/01bd0755da254d92943f799fe0cbf703, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/dada6e8006174dc9b6dcc4b4f173252b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7d34b868989f4441bc220b6355761d1d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7f456716be7040ed9fdf18e19eaa2d3a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/591cfd1c471242899cbd2c57eb9950ec, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8ea522b247754bce9aac2e769526e7aa, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/eed41972e9bd48ca9362db1c815a9040, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/fdd7b7cb72a54001b3c75ad3f6ad620d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5fbde6523ab3421cb937eb6b6a74ec5c] to archive 2024-11-15T04:40:39,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:40:39,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/20afd1e7e2dd4908a369dc8734635702 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/20afd1e7e2dd4908a369dc8734635702 2024-11-15T04:40:39,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9689c61b42c94973964391a37edfb40c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9689c61b42c94973964391a37edfb40c 2024-11-15T04:40:39,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2deaacbcb22c458b82dc8b9bb59c9184 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2deaacbcb22c458b82dc8b9bb59c9184 2024-11-15T04:40:39,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8beb2789990c42cf8a8ad529b31e25c3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8beb2789990c42cf8a8ad529b31e25c3 2024-11-15T04:40:39,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ce09d5b6ae714b0d9a83b5981d05efb1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ce09d5b6ae714b0d9a83b5981d05efb1 2024-11-15T04:40:39,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5ca62f0085574f7cb4690af1a7fb25a3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5ca62f0085574f7cb4690af1a7fb25a3 2024-11-15T04:40:39,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/f457e88170db4c5ba769894c1a01ce65 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/f457e88170db4c5ba769894c1a01ce65 2024-11-15T04:40:39,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/081ef298f95d4b18a30b99ab1a909695 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/081ef298f95d4b18a30b99ab1a909695 2024-11-15T04:40:39,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/c74f4c3978134a5383932829e44aad1b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/c74f4c3978134a5383932829e44aad1b 2024-11-15T04:40:39,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9b4668b2bd7345ab9e99f4b1c298a19e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/9b4668b2bd7345ab9e99f4b1c298a19e 2024-11-15T04:40:39,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ddbbf08e9c404d2a966d38cc1001d566 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/ddbbf08e9c404d2a966d38cc1001d566 2024-11-15T04:40:39,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/229326c7741041d59851182ca2e1885b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/229326c7741041d59851182ca2e1885b 2024-11-15T04:40:39,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/36a146b810b8494f80ce58859749831d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/36a146b810b8494f80ce58859749831d 2024-11-15T04:40:39,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/6e9654ca510d45b1863a07272badd03f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/6e9654ca510d45b1863a07272badd03f 2024-11-15T04:40:39,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/438d5b28b7c24358a202b3111510c5ed to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/438d5b28b7c24358a202b3111510c5ed 2024-11-15T04:40:39,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/391fa96d1d8c462688e162d4ebbe1758 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/391fa96d1d8c462688e162d4ebbe1758 2024-11-15T04:40:39,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/e8bc5b4580154a5ab84f3bdbb5fa08da to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/e8bc5b4580154a5ab84f3bdbb5fa08da 2024-11-15T04:40:39,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/32c2f33265bd487f9377e06f5929b06e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/32c2f33265bd487f9377e06f5929b06e 2024-11-15T04:40:39,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2ceaa8424b3840d186c748e8e8b619a5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/2ceaa8424b3840d186c748e8e8b619a5 2024-11-15T04:40:39,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d454ca6f0d3347f5b23289191a1bc8bf to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d454ca6f0d3347f5b23289191a1bc8bf 2024-11-15T04:40:39,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/714ba4fb5c80491c8a4f059ca27250c3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/714ba4fb5c80491c8a4f059ca27250c3 2024-11-15T04:40:39,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d0d7aecfa9b34f05a36d984ee8e6b151 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/d0d7aecfa9b34f05a36d984ee8e6b151 2024-11-15T04:40:39,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/01bd0755da254d92943f799fe0cbf703 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/01bd0755da254d92943f799fe0cbf703 2024-11-15T04:40:39,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/dada6e8006174dc9b6dcc4b4f173252b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/dada6e8006174dc9b6dcc4b4f173252b 2024-11-15T04:40:39,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7d34b868989f4441bc220b6355761d1d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7d34b868989f4441bc220b6355761d1d 2024-11-15T04:40:39,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7f456716be7040ed9fdf18e19eaa2d3a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/7f456716be7040ed9fdf18e19eaa2d3a 2024-11-15T04:40:39,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/591cfd1c471242899cbd2c57eb9950ec to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/591cfd1c471242899cbd2c57eb9950ec 2024-11-15T04:40:39,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8ea522b247754bce9aac2e769526e7aa to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/8ea522b247754bce9aac2e769526e7aa 2024-11-15T04:40:39,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/eed41972e9bd48ca9362db1c815a9040 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/eed41972e9bd48ca9362db1c815a9040 2024-11-15T04:40:39,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/fdd7b7cb72a54001b3c75ad3f6ad620d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/fdd7b7cb72a54001b3c75ad3f6ad620d 2024-11-15T04:40:39,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5fbde6523ab3421cb937eb6b6a74ec5c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/5fbde6523ab3421cb937eb6b6a74ec5c 2024-11-15T04:40:39,718 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/recovered.edits/464.seqid, newMaxSeqId=464, maxSeqId=4 2024-11-15T04:40:39,719 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. 2024-11-15T04:40:39,720 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 465c1abe9e468bea9e6a5aa6fe32be31: Waiting for close lock at 1731645637879Running coprocessor pre-close hooks at 1731645637879Disabling compacts and flushes for region at 1731645637879Disabling writes for close at 1731645637879Obtaining lock to block concurrent updates at 1731645637880 (+1 ms)Preparing flush snapshotting stores in 465c1abe9e468bea9e6a5aa6fe32be31 at 1731645637880Finished memstore snapshotting TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1731645637880Flushing stores of TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31. at 1731645637880Flushing 465c1abe9e468bea9e6a5aa6fe32be31/A: creating writer at 1731645637880Flushing mob file 465c1abe9e468bea9e6a5aa6fe32be31/A: appending metadata at 1731645637885 (+5 ms)Flushing mob file 465c1abe9e468bea9e6a5aa6fe32be31/A: closing flushed file at 1731645637885Flushing 465c1abe9e468bea9e6a5aa6fe32be31/A: appending metadata at 1731645638295 (+410 ms)Flushing 465c1abe9e468bea9e6a5aa6fe32be31/A: closing flushed file at 1731645638295Flushing 465c1abe9e468bea9e6a5aa6fe32be31/B: creating writer at 1731645638704 (+409 ms)Flushing 465c1abe9e468bea9e6a5aa6fe32be31/B: appending metadata at 1731645638707 (+3 ms)Flushing 465c1abe9e468bea9e6a5aa6fe32be31/B: closing flushed file at 1731645638707Flushing 465c1abe9e468bea9e6a5aa6fe32be31/C: creating writer at 1731645639120 (+413 ms)Flushing 465c1abe9e468bea9e6a5aa6fe32be31/C: appending metadata at 1731645639123 (+3 ms)Flushing 465c1abe9e468bea9e6a5aa6fe32be31/C: closing flushed file at 1731645639123Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b15021: reopening flushed file at 1731645639533 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3386ed44: reopening flushed file at 1731645639540 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f9076d1: reopening flushed file at 1731645639545 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 465c1abe9e468bea9e6a5aa6fe32be31 in 1673ms, sequenceid=461, compaction requested=false at 1731645639552 (+7 ms)Writing region close event to WAL at 1731645639711 (+159 ms)Running coprocessor post-close hooks at 1731645639719 (+8 ms)Closed at 1731645639719 2024-11-15T04:40:39,722 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,723 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=465c1abe9e468bea9e6a5aa6fe32be31, regionState=CLOSED 2024-11-15T04:40:39,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:40:39,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-15T04:40:39,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 465c1abe9e468bea9e6a5aa6fe32be31, server=26747e6ada4f,42935,1731645581086 in 2.0010 sec 2024-11-15T04:40:39,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-15T04:40:39,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=465c1abe9e468bea9e6a5aa6fe32be31, UNASSIGN in 2.0070 sec 2024-11-15T04:40:39,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T04:40:39,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0150 sec 2024-11-15T04:40:39,740 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645639740"}]},"ts":"1731645639740"} 2024-11-15T04:40:39,741 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T04:40:39,741 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T04:40:39,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0270 sec 2024-11-15T04:40:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T04:40:39,873 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:39,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T04:40:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:39,877 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:39,878 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T04:40:39,883 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,886 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/recovered.edits] 2024-11-15T04:40:39,889 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/27a8716b70a840cd83519b73bc9a8b3c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/27a8716b70a840cd83519b73bc9a8b3c 2024-11-15T04:40:39,890 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/697de655ec3842b6ba8f38fd6624ab22 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/A/697de655ec3842b6ba8f38fd6624ab22 2024-11-15T04:40:39,892 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/37b61ea6d87749939cac61d1878de89a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/37b61ea6d87749939cac61d1878de89a 2024-11-15T04:40:39,893 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/cc32f44df1f64fec9742b101408b67f2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/B/cc32f44df1f64fec9742b101408b67f2 2024-11-15T04:40:39,896 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/447b60a7228449a9be08641e887058ce to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/447b60a7228449a9be08641e887058ce 2024-11-15T04:40:39,898 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/6a90cb198bc34fcbbb00d1fa37153d04 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/C/6a90cb198bc34fcbbb00d1fa37153d04 2024-11-15T04:40:39,903 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/recovered.edits/464.seqid to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31/recovered.edits/464.seqid 2024-11-15T04:40:39,904 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,904 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T04:40:39,904 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T04:40:39,905 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T04:40:39,909 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150a2639a9bc04411188d608ea7a52e9cc_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150a2639a9bc04411188d608ea7a52e9cc_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,911 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115276e12b21290490c9dfe599ffc946366_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115276e12b21290490c9dfe599ffc946366_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,912 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152a5fa02f1ec844268fe6d3ad7bcc280b_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152a5fa02f1ec844268fe6d3ad7bcc280b_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,913 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152bf9bec0e94441ccb08dceca218256d0_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152bf9bec0e94441ccb08dceca218256d0_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,915 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a1910a16a2c44f395adaf91ec5d79c3_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a1910a16a2c44f395adaf91ec5d79c3_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,916 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153bac23c61dbe428aa053fb7803ae07f5_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153bac23c61dbe428aa053fb7803ae07f5_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,918 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153dad663b473642798793b82fe235b160_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153dad663b473642798793b82fe235b160_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,919 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153eb107d493614928859c254d9517198b_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153eb107d493614928859c254d9517198b_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,920 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115442c648da42c4ae7a2c924bdbec0213c_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115442c648da42c4ae7a2c924bdbec0213c_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,922 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155630a5d300bf4c91b42af0150ae0a4d4_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155630a5d300bf4c91b42af0150ae0a4d4_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,923 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155848f2290a7d41d49732ab7ea6bc2565_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155848f2290a7d41d49732ab7ea6bc2565_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,924 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155b5dbcebe39642c6b9e4246bd4fa5f93_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155b5dbcebe39642c6b9e4246bd4fa5f93_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,925 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156374c3e0960146aa9445fa69074fb918_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156374c3e0960146aa9445fa69074fb918_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,927 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111566c39e620e91460693daa10ad6f451c5_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111566c39e620e91460693daa10ad6f451c5_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,928 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156ea00dad30b44d6b92312908c9463f21_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156ea00dad30b44d6b92312908c9463f21_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,930 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158b000b4a245b4b209e8072d51f4bc044_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158b000b4a245b4b209e8072d51f4bc044_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,931 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158c66426e79004efc927b8fcd7a3b5a49_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158c66426e79004efc927b8fcd7a3b5a49_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,933 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a0f70a08fa0b4fdd91bd59306cb69058_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a0f70a08fa0b4fdd91bd59306cb69058_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,935 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a13fbe82320948cf9868f191f1d0086d_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a13fbe82320948cf9868f191f1d0086d_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,937 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a382aaab5613462eacdb860198be7135_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a382aaab5613462eacdb860198be7135_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,940 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d2297fc41092413b84eacfb88a927b2c_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d2297fc41092413b84eacfb88a927b2c_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,941 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d81ebbac587145b5b4b8abe574363527_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d81ebbac587145b5b4b8abe574363527_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,942 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115def97644733d457992b3c813eece6c8b_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115def97644733d457992b3c813eece6c8b_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,944 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ec5088faf96f489897ca94ee760706e3_465c1abe9e468bea9e6a5aa6fe32be31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ec5088faf96f489897ca94ee760706e3_465c1abe9e468bea9e6a5aa6fe32be31 2024-11-15T04:40:39,944 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T04:40:39,947 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:39,949 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T04:40:39,953 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T04:40:39,955 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:39,955 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T04:40:39,955 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731645639955"}]},"ts":"9223372036854775807"} 2024-11-15T04:40:39,958 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T04:40:39,958 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 465c1abe9e468bea9e6a5aa6fe32be31, NAME => 'TestAcidGuarantees,,1731645612273.465c1abe9e468bea9e6a5aa6fe32be31.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T04:40:39,958 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T04:40:39,958 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731645639958"}]},"ts":"9223372036854775807"} 2024-11-15T04:40:39,961 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T04:40:39,962 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:39,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 88 msec 2024-11-15T04:40:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T04:40:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T04:40:39,985 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:39,996 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 240) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/cluster_a5d1cf9b-a665-2596-bed6-8ed0e5e2e7a7/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1761888392_22 at /127.0.0.1:40982 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1761888392_22 at /127.0.0.1:52520 [Waiting for operation #276] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/cluster_a5d1cf9b-a665-2596-bed6-8ed0e5e2e7a7/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1356923970_22 at /127.0.0.1:40840 [Waiting for operation #325] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1356923970_22 at /127.0.0.1:43666 [Waiting for operation #59] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=529 (was 433) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=14682 (was 15261) 2024-11-15T04:40:40,006 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=529, ProcessCount=11, AvailableMemoryMB=14681 2024-11-15T04:40:40,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T04:40:40,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T04:40:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T04:40:40,010 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T04:40:40,011 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:40,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-15T04:40:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-15T04:40:40,012 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T04:40:40,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742074_1250 (size=959) 2024-11-15T04:40:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-15T04:40:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-15T04:40:40,420 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6 2024-11-15T04:40:40,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742075_1251 (size=53) 2024-11-15T04:40:40,637 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T04:40:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-15T04:40:40,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:40:40,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e5b59d814d067d3a48ee3af2799f7c44, disabling compactions & flushes 2024-11-15T04:40:40,827 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:40,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:40,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. after waiting 0 ms 2024-11-15T04:40:40,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:40,827 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:40,827 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e5b59d814d067d3a48ee3af2799f7c44: Waiting for close lock at 1731645640827Disabling compacts and flushes for region at 1731645640827Disabling writes for close at 1731645640827Writing region close event to WAL at 1731645640827Closed at 1731645640827 2024-11-15T04:40:40,828 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T04:40:40,829 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731645640828"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731645640828"}]},"ts":"1731645640828"} 2024-11-15T04:40:40,830 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T04:40:40,831 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T04:40:40,831 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645640831"}]},"ts":"1731645640831"} 2024-11-15T04:40:40,832 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T04:40:40,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5b59d814d067d3a48ee3af2799f7c44, ASSIGN}] 2024-11-15T04:40:40,833 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5b59d814d067d3a48ee3af2799f7c44, ASSIGN 2024-11-15T04:40:40,834 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5b59d814d067d3a48ee3af2799f7c44, ASSIGN; state=OFFLINE, location=26747e6ada4f,42935,1731645581086; forceNewPlan=false, retain=false 2024-11-15T04:40:40,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=e5b59d814d067d3a48ee3af2799f7c44, regionState=OPENING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:40,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5b59d814d067d3a48ee3af2799f7c44, ASSIGN because future has completed 2024-11-15T04:40:40,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:40:41,142 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:41,142 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:40:41,143 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,143 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:40:41,143 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,143 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,146 INFO [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,149 INFO [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5b59d814d067d3a48ee3af2799f7c44 columnFamilyName A 2024-11-15T04:40:41,149 DEBUG [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:41,150 INFO [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] regionserver.HStore(327): Store=e5b59d814d067d3a48ee3af2799f7c44/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:40:41,150 INFO [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,152 INFO [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5b59d814d067d3a48ee3af2799f7c44 columnFamilyName B 2024-11-15T04:40:41,152 DEBUG [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-15T04:40:41,153 INFO [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] regionserver.HStore(327): Store=e5b59d814d067d3a48ee3af2799f7c44/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:40:41,154 INFO [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,155 INFO [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5b59d814d067d3a48ee3af2799f7c44 columnFamilyName C 2024-11-15T04:40:41,155 DEBUG [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:40:41,155 INFO [StoreOpener-e5b59d814d067d3a48ee3af2799f7c44-1 {}] regionserver.HStore(327): Store=e5b59d814d067d3a48ee3af2799f7c44/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:40:41,156 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,156 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,157 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,158 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,158 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,158 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:40:41,159 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,161 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T04:40:41,162 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened e5b59d814d067d3a48ee3af2799f7c44; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59839726, jitterRate=-0.10831859707832336}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:40:41,162 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:41,162 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for e5b59d814d067d3a48ee3af2799f7c44: Running coprocessor pre-open hook at 1731645641144Writing region info on filesystem at 1731645641144Initializing all the Stores at 1731645641145 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645641145Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645641146 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645641146Cleaning up temporary data from old regions at 1731645641158 (+12 ms)Running coprocessor post-open hooks at 1731645641162 (+4 ms)Region opened successfully at 1731645641162 2024-11-15T04:40:41,163 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., pid=59, masterSystemTime=1731645641138 2024-11-15T04:40:41,165 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:41,165 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:41,165 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=e5b59d814d067d3a48ee3af2799f7c44, regionState=OPEN, openSeqNum=2, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:41,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:40:41,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T04:40:41,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 in 181 msec 2024-11-15T04:40:41,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-15T04:40:41,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5b59d814d067d3a48ee3af2799f7c44, ASSIGN in 337 msec 2024-11-15T04:40:41,172 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T04:40:41,172 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645641172"}]},"ts":"1731645641172"} 2024-11-15T04:40:41,174 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T04:40:41,175 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T04:40:41,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-15T04:40:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-15T04:40:42,164 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:42,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43dfb4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,166 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18667d1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,168 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4905445e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,169 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,173 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T04:40:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 26747e6ada4f,42655,1731645580243 2024-11-15T04:40:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@37284d9b 2024-11-15T04:40:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T04:40:42,175 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T04:40:42,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ddfbea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,178 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54344d2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,179 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53896ed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fbbb442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,181 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b0f8044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,182 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b33233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341051a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,184 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109f9f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,185 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3355ad4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309aa0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,186 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,187 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cdff61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,188 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ce1685, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6261277b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,190 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10508f64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,192 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5ed280, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106f6d84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,194 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1271122c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,195 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34608aec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a00a41f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,197 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce2e4a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f38d64a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,199 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a680e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,199 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,199 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,200 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162041d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,202 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d52262a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292fd0b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,203 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73789554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,205 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc47c1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,206 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710fcbd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,206 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:40:42,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:40:42,207 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:40:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:40:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:40:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2365e90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:40:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:40:42,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:40:42,208 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:40:42,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77422557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:40:42,209 DEBUG [Thread-1182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,209 DEBUG [Thread-1180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,209 DEBUG [Thread-1178 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,209 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,209 DEBUG [Thread-1169 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,209 DEBUG [Thread-1174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,210 DEBUG [Thread-1165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,210 DEBUG [Thread-1178 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,210 DEBUG [Thread-1182 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,210 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:42,210 DEBUG [Thread-1171 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,210 DEBUG [Thread-1178 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,210 DEBUG [Thread-1182 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,211 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,212 DEBUG [Thread-1167 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,213 DEBUG [Thread-1176 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:40:42,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:40:42,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:40:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-15T04:40:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:40:42,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:40:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:40:42,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:40:42,216 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T04:40:42,217 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:40:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:40:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:40:42,218 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:40:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:40:42,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:40:42,220 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:42,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:42,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:40:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:42,225 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:42,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:40:42,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/b033061e768d466ba2c9e74bd347628a is 50, key is test_row_0/A:col10/1731645642223/Put/seqid=0 2024-11-15T04:40:42,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645702259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645702261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742076_1252 (size=12001) 2024-11-15T04:40:42,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645702261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/b033061e768d466ba2c9e74bd347628a 2024-11-15T04:40:42,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645702262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645702267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/07a49cbcdc3b4c928d4bea6a321c59d4 is 50, key is test_row_0/B:col10/1731645642223/Put/seqid=0 2024-11-15T04:40:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T04:40:42,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742077_1253 (size=12001) 2024-11-15T04:40:42,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/07a49cbcdc3b4c928d4bea6a321c59d4 2024-11-15T04:40:42,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T04:40:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:42,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:42,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645702373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645702373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645702373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645702373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/e762ecac47524d14a21052328ede9c0b is 50, key is test_row_0/C:col10/1731645642223/Put/seqid=0 2024-11-15T04:40:42,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645702382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742078_1254 (size=12001) 2024-11-15T04:40:42,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/e762ecac47524d14a21052328ede9c0b 2024-11-15T04:40:42,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/b033061e768d466ba2c9e74bd347628a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b033061e768d466ba2c9e74bd347628a 2024-11-15T04:40:42,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b033061e768d466ba2c9e74bd347628a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T04:40:42,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/07a49cbcdc3b4c928d4bea6a321c59d4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/07a49cbcdc3b4c928d4bea6a321c59d4 2024-11-15T04:40:42,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/07a49cbcdc3b4c928d4bea6a321c59d4, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T04:40:42,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/e762ecac47524d14a21052328ede9c0b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e762ecac47524d14a21052328ede9c0b 2024-11-15T04:40:42,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e762ecac47524d14a21052328ede9c0b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T04:40:42,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e5b59d814d067d3a48ee3af2799f7c44 in 206ms, sequenceid=12, compaction requested=false 2024-11-15T04:40:42,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:42,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-15T04:40:42,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:42,527 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:40:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T04:40:42,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d219ea45a6224438925e029f0eac5b0e is 50, key is test_row_0/A:col10/1731645642259/Put/seqid=0 2024-11-15T04:40:42,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742079_1255 (size=12001) 2024-11-15T04:40:42,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:42,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645702593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645702593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645702595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645702595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645702596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645702703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645702703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645702703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645702703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T04:40:42,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645702903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645702912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645702912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645702912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645702912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,005 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d219ea45a6224438925e029f0eac5b0e 2024-11-15T04:40:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/92e6f03b9f134d738aaac5571c02dbeb is 50, key is test_row_0/B:col10/1731645642259/Put/seqid=0 2024-11-15T04:40:43,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742080_1256 (size=12001) 2024-11-15T04:40:43,038 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/92e6f03b9f134d738aaac5571c02dbeb 2024-11-15T04:40:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/75292478ddb145dbb7d3b1b73f6b7bb9 is 50, key is test_row_0/C:col10/1731645642259/Put/seqid=0 2024-11-15T04:40:43,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742081_1257 (size=12001) 2024-11-15T04:40:43,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645703222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645703223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645703223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645703223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T04:40:43,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645703413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,453 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/75292478ddb145dbb7d3b1b73f6b7bb9 2024-11-15T04:40:43,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d219ea45a6224438925e029f0eac5b0e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d219ea45a6224438925e029f0eac5b0e 2024-11-15T04:40:43,461 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d219ea45a6224438925e029f0eac5b0e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T04:40:43,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/92e6f03b9f134d738aaac5571c02dbeb as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/92e6f03b9f134d738aaac5571c02dbeb 2024-11-15T04:40:43,467 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/92e6f03b9f134d738aaac5571c02dbeb, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T04:40:43,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/75292478ddb145dbb7d3b1b73f6b7bb9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/75292478ddb145dbb7d3b1b73f6b7bb9 2024-11-15T04:40:43,472 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/75292478ddb145dbb7d3b1b73f6b7bb9, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T04:40:43,473 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for e5b59d814d067d3a48ee3af2799f7c44 in 946ms, sequenceid=37, compaction requested=false 2024-11-15T04:40:43,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:43,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:43,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-15T04:40:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-15T04:40:43,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T04:40:43,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2540 sec 2024-11-15T04:40:43,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.2660 sec 2024-11-15T04:40:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:43,735 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:43,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T04:40:43,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/849acc3ec8a44c158d24e52f7b86b476 is 50, key is test_row_0/A:col10/1731645643735/Put/seqid=0 2024-11-15T04:40:43,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742082_1258 (size=14341) 2024-11-15T04:40:43,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645703785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645703786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645703787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645703787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645703893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645703894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645703894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:43,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645703896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:43,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645704103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645704103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645704104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645704104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/849acc3ec8a44c158d24e52f7b86b476 2024-11-15T04:40:44,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/4f841926f0474d11ab6b55052ea0552f is 50, key is test_row_0/B:col10/1731645643735/Put/seqid=0 2024-11-15T04:40:44,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742083_1259 (size=12001) 2024-11-15T04:40:44,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/4f841926f0474d11ab6b55052ea0552f 2024-11-15T04:40:44,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/f172e84322cf44169a89403ae8edeae7 is 50, key is test_row_0/C:col10/1731645643735/Put/seqid=0 2024-11-15T04:40:44,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742084_1260 (size=12001) 2024-11-15T04:40:44,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/f172e84322cf44169a89403ae8edeae7 2024-11-15T04:40:44,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/849acc3ec8a44c158d24e52f7b86b476 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/849acc3ec8a44c158d24e52f7b86b476 2024-11-15T04:40:44,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/849acc3ec8a44c158d24e52f7b86b476, entries=200, sequenceid=52, filesize=14.0 K 2024-11-15T04:40:44,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/4f841926f0474d11ab6b55052ea0552f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/4f841926f0474d11ab6b55052ea0552f 2024-11-15T04:40:44,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/4f841926f0474d11ab6b55052ea0552f, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T04:40:44,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/f172e84322cf44169a89403ae8edeae7 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/f172e84322cf44169a89403ae8edeae7 2024-11-15T04:40:44,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/f172e84322cf44169a89403ae8edeae7, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T04:40:44,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e5b59d814d067d3a48ee3af2799f7c44 in 476ms, sequenceid=52, compaction requested=true 2024-11-15T04:40:44,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:44,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:44,211 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:44,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:44,211 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:44,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:44,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:44,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:44,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:44,214 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:44,214 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:44,214 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:40:44,214 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:44,214 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:40:44,214 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:44,214 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/07a49cbcdc3b4c928d4bea6a321c59d4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/92e6f03b9f134d738aaac5571c02dbeb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/4f841926f0474d11ab6b55052ea0552f] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=35.2 K 2024-11-15T04:40:44,214 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b033061e768d466ba2c9e74bd347628a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d219ea45a6224438925e029f0eac5b0e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/849acc3ec8a44c158d24e52f7b86b476] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=37.4 K 2024-11-15T04:40:44,215 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b033061e768d466ba2c9e74bd347628a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731645642218 2024-11-15T04:40:44,215 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 07a49cbcdc3b4c928d4bea6a321c59d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731645642218 2024-11-15T04:40:44,215 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d219ea45a6224438925e029f0eac5b0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731645642251 2024-11-15T04:40:44,215 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e6f03b9f134d738aaac5571c02dbeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731645642251 2024-11-15T04:40:44,216 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f841926f0474d11ab6b55052ea0552f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645643733 2024-11-15T04:40:44,216 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 849acc3ec8a44c158d24e52f7b86b476, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645642590 2024-11-15T04:40:44,231 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:44,231 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:44,231 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/5568e9a37abd415db72e230668988f6a is 50, key is test_row_0/B:col10/1731645643735/Put/seqid=0 2024-11-15T04:40:44,231 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/1e02103320434a92b9e253466bd44a26 is 50, key is test_row_0/A:col10/1731645643735/Put/seqid=0 2024-11-15T04:40:44,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742086_1262 (size=12104) 2024-11-15T04:40:44,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742085_1261 (size=12104) 2024-11-15T04:40:44,253 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/5568e9a37abd415db72e230668988f6a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/5568e9a37abd415db72e230668988f6a 2024-11-15T04:40:44,257 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into 5568e9a37abd415db72e230668988f6a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:44,257 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:44,257 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=13, startTime=1731645644211; duration=0sec 2024-11-15T04:40:44,257 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:44,257 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:40:44,257 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:44,259 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:44,259 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/C is initiating minor compaction (all files) 2024-11-15T04:40:44,259 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/C in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:44,259 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e762ecac47524d14a21052328ede9c0b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/75292478ddb145dbb7d3b1b73f6b7bb9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/f172e84322cf44169a89403ae8edeae7] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=35.2 K 2024-11-15T04:40:44,259 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e762ecac47524d14a21052328ede9c0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731645642218 2024-11-15T04:40:44,260 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 75292478ddb145dbb7d3b1b73f6b7bb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731645642251 2024-11-15T04:40:44,260 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting f172e84322cf44169a89403ae8edeae7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645643733 2024-11-15T04:40:44,268 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:44,268 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/efe44a39d9004b9984b2c8c7a3b713b0 is 50, key is test_row_0/C:col10/1731645643735/Put/seqid=0 2024-11-15T04:40:44,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742087_1263 (size=12104) 2024-11-15T04:40:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-15T04:40:44,363 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:44,363 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-15T04:40:44,366 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T04:40:44,367 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:44,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:44,416 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:44,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T04:40:44,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/3d1cd57166244d338d9f5c96bc668ecf is 50, key is test_row_0/A:col10/1731645643784/Put/seqid=0 2024-11-15T04:40:44,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742088_1264 (size=12001) 2024-11-15T04:40:44,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/3d1cd57166244d338d9f5c96bc668ecf 2024-11-15T04:40:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645704433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645704441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645704444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645704446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645704446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/50717a862d7d41638e8effba492589df is 50, key is test_row_0/B:col10/1731645643784/Put/seqid=0 2024-11-15T04:40:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T04:40:44,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T04:40:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:44,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:44,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742089_1265 (size=12001) 2024-11-15T04:40:44,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:44,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:44,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/50717a862d7d41638e8effba492589df 2024-11-15T04:40:44,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/edcd9b95c68747b9999f7e2d9f9ace9a is 50, key is test_row_0/C:col10/1731645643784/Put/seqid=0 2024-11-15T04:40:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645704553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645704553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645704562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645704563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645704563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742090_1266 (size=12001) 2024-11-15T04:40:44,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/edcd9b95c68747b9999f7e2d9f9ace9a 2024-11-15T04:40:44,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/3d1cd57166244d338d9f5c96bc668ecf as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/3d1cd57166244d338d9f5c96bc668ecf 2024-11-15T04:40:44,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/3d1cd57166244d338d9f5c96bc668ecf, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T04:40:44,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/50717a862d7d41638e8effba492589df as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/50717a862d7d41638e8effba492589df 2024-11-15T04:40:44,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/50717a862d7d41638e8effba492589df, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T04:40:44,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/edcd9b95c68747b9999f7e2d9f9ace9a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/edcd9b95c68747b9999f7e2d9f9ace9a 2024-11-15T04:40:44,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/edcd9b95c68747b9999f7e2d9f9ace9a, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T04:40:44,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e5b59d814d067d3a48ee3af2799f7c44 in 206ms, sequenceid=75, compaction requested=false 2024-11-15T04:40:44,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:44,656 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/1e02103320434a92b9e253466bd44a26 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1e02103320434a92b9e253466bd44a26 2024-11-15T04:40:44,668 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into 1e02103320434a92b9e253466bd44a26(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:44,668 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:44,668 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=13, startTime=1731645644211; duration=0sec 2024-11-15T04:40:44,668 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:44,668 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:40:44,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-15T04:40:44,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:44,678 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T04:40:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T04:40:44,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/856141af95ff41b893ec4caec0ba4586 is 50, key is test_row_0/A:col10/1731645644422/Put/seqid=0 2024-11-15T04:40:44,688 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/efe44a39d9004b9984b2c8c7a3b713b0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/efe44a39d9004b9984b2c8c7a3b713b0 2024-11-15T04:40:44,698 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/C of e5b59d814d067d3a48ee3af2799f7c44 into efe44a39d9004b9984b2c8c7a3b713b0(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:44,698 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:44,698 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/C, priority=13, startTime=1731645644211; duration=0sec 2024-11-15T04:40:44,698 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:44,699 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:40:44,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742091_1267 (size=12001) 2024-11-15T04:40:44,730 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/856141af95ff41b893ec4caec0ba4586 2024-11-15T04:40:44,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/a8fb07064a7d427390135dd590323ee6 is 50, key is test_row_0/B:col10/1731645644422/Put/seqid=0 2024-11-15T04:40:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:44,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:44,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742092_1268 (size=12001) 2024-11-15T04:40:44,791 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/a8fb07064a7d427390135dd590323ee6 2024-11-15T04:40:44,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/cbab6f58be554914a4a1b7394d802d02 is 50, key is test_row_0/C:col10/1731645644422/Put/seqid=0 2024-11-15T04:40:44,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645704817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645704817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645704819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645704819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645704825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742093_1269 (size=12001) 2024-11-15T04:40:44,839 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/cbab6f58be554914a4a1b7394d802d02 2024-11-15T04:40:44,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/856141af95ff41b893ec4caec0ba4586 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/856141af95ff41b893ec4caec0ba4586 2024-11-15T04:40:44,853 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/856141af95ff41b893ec4caec0ba4586, entries=150, sequenceid=90, filesize=11.7 K 2024-11-15T04:40:44,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/a8fb07064a7d427390135dd590323ee6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/a8fb07064a7d427390135dd590323ee6 2024-11-15T04:40:44,859 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/a8fb07064a7d427390135dd590323ee6, entries=150, sequenceid=90, filesize=11.7 K 2024-11-15T04:40:44,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/cbab6f58be554914a4a1b7394d802d02 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cbab6f58be554914a4a1b7394d802d02 2024-11-15T04:40:44,872 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cbab6f58be554914a4a1b7394d802d02, entries=150, sequenceid=90, filesize=11.7 K 2024-11-15T04:40:44,874 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e5b59d814d067d3a48ee3af2799f7c44 in 197ms, sequenceid=90, compaction requested=true 2024-11-15T04:40:44,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:44,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:44,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-15T04:40:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-15T04:40:44,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-15T04:40:44,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 510 msec 2024-11-15T04:40:44,882 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 518 msec 2024-11-15T04:40:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:44,937 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:44,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T04:40:44,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/1524457abdbf49928ab15481e5fe2059 is 50, key is test_row_0/A:col10/1731645644936/Put/seqid=0 2024-11-15T04:40:44,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645704945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742094_1270 (size=16681) 2024-11-15T04:40:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645704946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645704946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645704947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645704947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:44,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T04:40:44,993 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:44,993 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:44,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-15T04:40:44,995 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:44,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T04:40:44,996 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:44,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:45,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645705053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645705062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645705062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645705062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645705063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T04:40:45,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T04:40:45,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:45,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645705262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645705272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,276 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T04:40:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645705273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645705273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645705273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T04:40:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:45,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T04:40:45,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/1524457abdbf49928ab15481e5fe2059 2024-11-15T04:40:45,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/9008a8df281c489fba59beee3ef87242 is 50, key is test_row_0/B:col10/1731645644936/Put/seqid=0 2024-11-15T04:40:45,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742095_1271 (size=12001) 2024-11-15T04:40:45,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T04:40:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:45,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645705573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645705583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645705583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645705583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645705583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:45,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:45,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T04:40:45,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:45,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T04:40:45,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T04:40:45,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:45,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/9008a8df281c489fba59beee3ef87242 2024-11-15T04:40:45,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/5d45e3bfac044a59a26e34f13760aec9 is 50, key is test_row_0/C:col10/1731645644936/Put/seqid=0 2024-11-15T04:40:45,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742096_1272 (size=12001) 2024-11-15T04:40:45,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T04:40:45,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:45,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:45,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:46,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T04:40:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:46,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:46,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645706083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:46,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645706093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:46,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645706093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:46,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645706102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:46,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645706102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:46,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T04:40:46,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T04:40:46,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:46,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:46,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:46,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:46,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:46,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:46,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/5d45e3bfac044a59a26e34f13760aec9 2024-11-15T04:40:46,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/1524457abdbf49928ab15481e5fe2059 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1524457abdbf49928ab15481e5fe2059 2024-11-15T04:40:46,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1524457abdbf49928ab15481e5fe2059, entries=250, sequenceid=117, filesize=16.3 K 2024-11-15T04:40:46,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/9008a8df281c489fba59beee3ef87242 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9008a8df281c489fba59beee3ef87242 2024-11-15T04:40:46,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9008a8df281c489fba59beee3ef87242, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T04:40:46,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/5d45e3bfac044a59a26e34f13760aec9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/5d45e3bfac044a59a26e34f13760aec9 2024-11-15T04:40:46,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/5d45e3bfac044a59a26e34f13760aec9, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T04:40:46,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e5b59d814d067d3a48ee3af2799f7c44 in 1341ms, sequenceid=117, compaction requested=true 2024-11-15T04:40:46,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:46,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:46,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:46,278 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:46,278 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:46,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:46,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:46,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:46,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:46,280 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:46,280 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:46,280 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:40:46,280 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:40:46,281 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:46,281 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/5568e9a37abd415db72e230668988f6a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/50717a862d7d41638e8effba492589df, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/a8fb07064a7d427390135dd590323ee6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9008a8df281c489fba59beee3ef87242] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=47.0 K 2024-11-15T04:40:46,281 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:46,281 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1e02103320434a92b9e253466bd44a26, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/3d1cd57166244d338d9f5c96bc668ecf, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/856141af95ff41b893ec4caec0ba4586, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1524457abdbf49928ab15481e5fe2059] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=51.5 K 2024-11-15T04:40:46,282 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 5568e9a37abd415db72e230668988f6a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645643733 2024-11-15T04:40:46,282 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e02103320434a92b9e253466bd44a26, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645643733 2024-11-15T04:40:46,282 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 50717a862d7d41638e8effba492589df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731645643784 2024-11-15T04:40:46,282 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d1cd57166244d338d9f5c96bc668ecf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731645643784 2024-11-15T04:40:46,282 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting a8fb07064a7d427390135dd590323ee6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731645644422 2024-11-15T04:40:46,282 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 856141af95ff41b893ec4caec0ba4586, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731645644422 2024-11-15T04:40:46,283 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9008a8df281c489fba59beee3ef87242, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645644823 2024-11-15T04:40:46,283 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1524457abdbf49928ab15481e5fe2059, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645644823 2024-11-15T04:40:46,293 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#222 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:46,294 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/b758275ce6fb4329a04ab2fc662c5b5c is 50, key is test_row_0/B:col10/1731645644936/Put/seqid=0 2024-11-15T04:40:46,298 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:46,299 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/a95cddcee5254fb59dfb19907fb96584 is 50, key is test_row_0/A:col10/1731645644936/Put/seqid=0 2024-11-15T04:40:46,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742097_1273 (size=12241) 2024-11-15T04:40:46,314 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/b758275ce6fb4329a04ab2fc662c5b5c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b758275ce6fb4329a04ab2fc662c5b5c 2024-11-15T04:40:46,320 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into b758275ce6fb4329a04ab2fc662c5b5c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:46,321 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:46,321 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=12, startTime=1731645646278; duration=0sec 2024-11-15T04:40:46,321 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:46,321 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:40:46,321 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:46,324 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:46,324 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/C is initiating minor compaction (all files) 2024-11-15T04:40:46,324 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/C in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:46,325 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/efe44a39d9004b9984b2c8c7a3b713b0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/edcd9b95c68747b9999f7e2d9f9ace9a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cbab6f58be554914a4a1b7394d802d02, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/5d45e3bfac044a59a26e34f13760aec9] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=47.0 K 2024-11-15T04:40:46,325 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting efe44a39d9004b9984b2c8c7a3b713b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645643733 2024-11-15T04:40:46,326 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting edcd9b95c68747b9999f7e2d9f9ace9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731645643784 2024-11-15T04:40:46,326 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting cbab6f58be554914a4a1b7394d802d02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731645644422 2024-11-15T04:40:46,327 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d45e3bfac044a59a26e34f13760aec9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645644823 2024-11-15T04:40:46,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742098_1274 (size=12241) 2024-11-15T04:40:46,342 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/a95cddcee5254fb59dfb19907fb96584 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a95cddcee5254fb59dfb19907fb96584 2024-11-15T04:40:46,343 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#C#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:46,344 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2b10d68dad0f4785ab276f34bf752196 is 50, key is test_row_0/C:col10/1731645644936/Put/seqid=0 2024-11-15T04:40:46,361 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into a95cddcee5254fb59dfb19907fb96584(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:46,362 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:46,362 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=12, startTime=1731645646278; duration=0sec 2024-11-15T04:40:46,362 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:46,362 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:40:46,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-15T04:40:46,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:46,378 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T04:40:46,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742099_1275 (size=12241) 2024-11-15T04:40:46,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/88688b42c8524ca8afe8c0ce158d82eb is 50, key is test_row_0/A:col10/1731645644946/Put/seqid=0 2024-11-15T04:40:46,390 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2b10d68dad0f4785ab276f34bf752196 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2b10d68dad0f4785ab276f34bf752196 2024-11-15T04:40:46,399 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/C of e5b59d814d067d3a48ee3af2799f7c44 into 2b10d68dad0f4785ab276f34bf752196(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:46,400 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:46,400 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/C, priority=12, startTime=1731645646278; duration=0sec 2024-11-15T04:40:46,400 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:46,400 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:40:46,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742100_1276 (size=9657) 2024-11-15T04:40:46,418 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/88688b42c8524ca8afe8c0ce158d82eb 2024-11-15T04:40:46,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/b6e378322ea14a4e8e94ddf0120662d3 is 50, key is test_row_0/B:col10/1731645644946/Put/seqid=0 2024-11-15T04:40:46,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742101_1277 (size=9657) 2024-11-15T04:40:46,452 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/b6e378322ea14a4e8e94ddf0120662d3 2024-11-15T04:40:46,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/cfa985dc0e3c4df7be4e24f36bd1332c is 50, key is test_row_0/C:col10/1731645644946/Put/seqid=0 2024-11-15T04:40:46,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742102_1278 (size=9657) 2024-11-15T04:40:46,481 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/cfa985dc0e3c4df7be4e24f36bd1332c 2024-11-15T04:40:46,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/88688b42c8524ca8afe8c0ce158d82eb as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/88688b42c8524ca8afe8c0ce158d82eb 2024-11-15T04:40:46,498 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/88688b42c8524ca8afe8c0ce158d82eb, entries=100, sequenceid=129, filesize=9.4 K 2024-11-15T04:40:46,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/b6e378322ea14a4e8e94ddf0120662d3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b6e378322ea14a4e8e94ddf0120662d3 2024-11-15T04:40:46,514 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b6e378322ea14a4e8e94ddf0120662d3, entries=100, sequenceid=129, filesize=9.4 K 2024-11-15T04:40:46,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/cfa985dc0e3c4df7be4e24f36bd1332c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cfa985dc0e3c4df7be4e24f36bd1332c 2024-11-15T04:40:46,522 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cfa985dc0e3c4df7be4e24f36bd1332c, entries=100, sequenceid=129, filesize=9.4 K 2024-11-15T04:40:46,524 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for e5b59d814d067d3a48ee3af2799f7c44 in 146ms, sequenceid=129, compaction requested=false 2024-11-15T04:40:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-15T04:40:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-15T04:40:46,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-15T04:40:46,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5300 sec 2024-11-15T04:40:46,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5380 sec 2024-11-15T04:40:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:47,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:47,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:40:47,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/4143f70760a543d49058d7e44c1a5c9c is 50, key is test_row_0/A:col10/1731645647111/Put/seqid=0 2024-11-15T04:40:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-15T04:40:47,143 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:47,143 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-15T04:40:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T04:40:47,148 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:47,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:47,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:47,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645707154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645707156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645707159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645707161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645707161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742103_1279 (size=14541) 2024-11-15T04:40:47,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/4143f70760a543d49058d7e44c1a5c9c 2024-11-15T04:40:47,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/83d635553eb84264be2ba708795eb433 is 50, key is test_row_0/B:col10/1731645647111/Put/seqid=0 2024-11-15T04:40:47,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742104_1280 (size=12151) 2024-11-15T04:40:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T04:40:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645707273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645707273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645707273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645707274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645707274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T04:40:47,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:47,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:47,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:47,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T04:40:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:47,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:47,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T04:40:47,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645707484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645707484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645707484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645707488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645707488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T04:40:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:47,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:47,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/83d635553eb84264be2ba708795eb433 2024-11-15T04:40:47,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2725d18573264048af2048c385aa6481 is 50, key is test_row_0/C:col10/1731645647111/Put/seqid=0 2024-11-15T04:40:47,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742105_1281 (size=12151) 2024-11-15T04:40:47,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T04:40:47,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:47,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:47,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:47,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T04:40:47,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645707793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645707793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645707793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645707793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645707793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:47,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T04:40:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:47,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:48,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T04:40:48,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:48,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:48,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:48,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:48,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:48,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:48,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2725d18573264048af2048c385aa6481 2024-11-15T04:40:48,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/4143f70760a543d49058d7e44c1a5c9c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/4143f70760a543d49058d7e44c1a5c9c 2024-11-15T04:40:48,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/4143f70760a543d49058d7e44c1a5c9c, entries=200, sequenceid=142, filesize=14.2 K 2024-11-15T04:40:48,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/83d635553eb84264be2ba708795eb433 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83d635553eb84264be2ba708795eb433 2024-11-15T04:40:48,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83d635553eb84264be2ba708795eb433, entries=150, sequenceid=142, filesize=11.9 K 2024-11-15T04:40:48,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2725d18573264048af2048c385aa6481 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2725d18573264048af2048c385aa6481 2024-11-15T04:40:48,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2725d18573264048af2048c385aa6481, entries=150, sequenceid=142, filesize=11.9 K 2024-11-15T04:40:48,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for e5b59d814d067d3a48ee3af2799f7c44 in 1003ms, sequenceid=142, compaction requested=true 2024-11-15T04:40:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:48,118 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:48,118 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:48,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:48,119 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:48,119 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:48,119 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:40:48,119 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:40:48,119 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:48,119 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:48,119 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a95cddcee5254fb59dfb19907fb96584, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/88688b42c8524ca8afe8c0ce158d82eb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/4143f70760a543d49058d7e44c1a5c9c] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=35.6 K 2024-11-15T04:40:48,119 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b758275ce6fb4329a04ab2fc662c5b5c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b6e378322ea14a4e8e94ddf0120662d3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83d635553eb84264be2ba708795eb433] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=33.3 K 2024-11-15T04:40:48,119 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting b758275ce6fb4329a04ab2fc662c5b5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645644823 2024-11-15T04:40:48,119 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting a95cddcee5254fb59dfb19907fb96584, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645644823 2024-11-15T04:40:48,119 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e378322ea14a4e8e94ddf0120662d3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731645644945 2024-11-15T04:40:48,119 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88688b42c8524ca8afe8c0ce158d82eb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731645644945 2024-11-15T04:40:48,120 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d635553eb84264be2ba708795eb433, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731645647103 2024-11-15T04:40:48,120 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4143f70760a543d49058d7e44c1a5c9c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731645647103 2024-11-15T04:40:48,132 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:48,133 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/9d1fe5427dce4e8aa84fdf6ef15b37d0 is 50, key is test_row_0/B:col10/1731645647111/Put/seqid=0 2024-11-15T04:40:48,137 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:48,138 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/a31a294c0c1b46768f1a08fdb8181b41 is 50, key is test_row_0/A:col10/1731645647111/Put/seqid=0 2024-11-15T04:40:48,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742107_1283 (size=12493) 2024-11-15T04:40:48,151 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/a31a294c0c1b46768f1a08fdb8181b41 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a31a294c0c1b46768f1a08fdb8181b41 2024-11-15T04:40:48,156 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into a31a294c0c1b46768f1a08fdb8181b41(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:48,157 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:48,157 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=13, startTime=1731645648118; duration=0sec 2024-11-15T04:40:48,157 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:48,157 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:40:48,157 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:48,158 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:48,158 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/C is initiating minor compaction (all files) 2024-11-15T04:40:48,158 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/C in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:48,158 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2b10d68dad0f4785ab276f34bf752196, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cfa985dc0e3c4df7be4e24f36bd1332c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2725d18573264048af2048c385aa6481] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=33.3 K 2024-11-15T04:40:48,158 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b10d68dad0f4785ab276f34bf752196, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645644823 2024-11-15T04:40:48,159 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfa985dc0e3c4df7be4e24f36bd1332c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731645644945 2024-11-15T04:40:48,159 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2725d18573264048af2048c385aa6481, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731645647103 2024-11-15T04:40:48,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742106_1282 (size=12493) 2024-11-15T04:40:48,168 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:48,169 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/b3ca0075e8ea4d38b828971b10a47e49 is 50, key is test_row_0/C:col10/1731645647111/Put/seqid=0 2024-11-15T04:40:48,171 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/9d1fe5427dce4e8aa84fdf6ef15b37d0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9d1fe5427dce4e8aa84fdf6ef15b37d0 2024-11-15T04:40:48,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742108_1284 (size=12493) 2024-11-15T04:40:48,179 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into 9d1fe5427dce4e8aa84fdf6ef15b37d0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:48,179 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:48,179 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=13, startTime=1731645648118; duration=0sec 2024-11-15T04:40:48,179 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:48,179 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:40:48,185 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/b3ca0075e8ea4d38b828971b10a47e49 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/b3ca0075e8ea4d38b828971b10a47e49 2024-11-15T04:40:48,193 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/C of e5b59d814d067d3a48ee3af2799f7c44 into b3ca0075e8ea4d38b828971b10a47e49(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:48,193 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:48,193 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/C, priority=13, startTime=1731645648118; duration=0sec 2024-11-15T04:40:48,193 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:48,193 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:40:48,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T04:40:48,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:48,232 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:40:48,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/560c430b674544a28a10693374ce1c79 is 50, key is test_row_0/A:col10/1731645647158/Put/seqid=0 2024-11-15T04:40:48,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742109_1285 (size=12151) 2024-11-15T04:40:48,281 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/560c430b674544a28a10693374ce1c79 2024-11-15T04:40:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T04:40:48,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/203fc2ecbabe4907abe8b2768eb51178 is 50, key is test_row_0/B:col10/1731645647158/Put/seqid=0 2024-11-15T04:40:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:48,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:48,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742110_1286 (size=12151) 2024-11-15T04:40:48,312 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/203fc2ecbabe4907abe8b2768eb51178 2024-11-15T04:40:48,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645708313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645708314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645708314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645708315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/fa27a161676640e5bb66b1d115808be4 is 50, key is test_row_0/C:col10/1731645647158/Put/seqid=0 2024-11-15T04:40:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645708313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742111_1287 (size=12151) 2024-11-15T04:40:48,352 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/fa27a161676640e5bb66b1d115808be4 2024-11-15T04:40:48,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/560c430b674544a28a10693374ce1c79 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/560c430b674544a28a10693374ce1c79 2024-11-15T04:40:48,365 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/560c430b674544a28a10693374ce1c79, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T04:40:48,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/203fc2ecbabe4907abe8b2768eb51178 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/203fc2ecbabe4907abe8b2768eb51178 2024-11-15T04:40:48,391 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/203fc2ecbabe4907abe8b2768eb51178, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T04:40:48,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/fa27a161676640e5bb66b1d115808be4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/fa27a161676640e5bb66b1d115808be4 2024-11-15T04:40:48,400 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/fa27a161676640e5bb66b1d115808be4, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T04:40:48,401 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for e5b59d814d067d3a48ee3af2799f7c44 in 170ms, sequenceid=170, compaction requested=false 2024-11-15T04:40:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-15T04:40:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-15T04:40:48,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T04:40:48,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2530 sec 2024-11-15T04:40:48,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.2620 sec 2024-11-15T04:40:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:48,424 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:48,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T04:40:48,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/b77ad6462ec54bd49d74ed3d5f2bd589 is 50, key is test_row_0/A:col10/1731645648424/Put/seqid=0 2024-11-15T04:40:48,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742112_1288 (size=14541) 2024-11-15T04:40:48,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/b77ad6462ec54bd49d74ed3d5f2bd589 2024-11-15T04:40:48,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/974b83b3fd294e8e9eb4614ec6214aa0 is 50, key is test_row_0/B:col10/1731645648424/Put/seqid=0 2024-11-15T04:40:48,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742113_1289 (size=12151) 2024-11-15T04:40:48,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/974b83b3fd294e8e9eb4614ec6214aa0 2024-11-15T04:40:48,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645708458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645708458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645708465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645708466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2687a00026644a74907b8e7ded765cf6 is 50, key is test_row_0/C:col10/1731645648424/Put/seqid=0 2024-11-15T04:40:48,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742114_1290 (size=12151) 2024-11-15T04:40:48,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645708572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645708572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645708583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645708583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645708782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645708782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645708793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645708793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:48,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:48,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2687a00026644a74907b8e7ded765cf6 2024-11-15T04:40:48,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/b77ad6462ec54bd49d74ed3d5f2bd589 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b77ad6462ec54bd49d74ed3d5f2bd589 2024-11-15T04:40:48,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b77ad6462ec54bd49d74ed3d5f2bd589, entries=200, sequenceid=184, filesize=14.2 K 2024-11-15T04:40:48,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/974b83b3fd294e8e9eb4614ec6214aa0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/974b83b3fd294e8e9eb4614ec6214aa0 2024-11-15T04:40:48,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/974b83b3fd294e8e9eb4614ec6214aa0, entries=150, sequenceid=184, filesize=11.9 K 2024-11-15T04:40:48,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2687a00026644a74907b8e7ded765cf6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2687a00026644a74907b8e7ded765cf6 2024-11-15T04:40:48,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2687a00026644a74907b8e7ded765cf6, entries=150, sequenceid=184, filesize=11.9 K 2024-11-15T04:40:48,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e5b59d814d067d3a48ee3af2799f7c44 in 495ms, sequenceid=184, compaction requested=true 2024-11-15T04:40:48,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:48,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:48,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:48,919 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:48,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:48,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:48,919 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:48,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:48,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:48,920 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:48,920 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:48,920 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:40:48,920 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:40:48,920 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:48,920 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:48,920 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a31a294c0c1b46768f1a08fdb8181b41, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/560c430b674544a28a10693374ce1c79, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b77ad6462ec54bd49d74ed3d5f2bd589] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=38.3 K 2024-11-15T04:40:48,920 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9d1fe5427dce4e8aa84fdf6ef15b37d0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/203fc2ecbabe4907abe8b2768eb51178, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/974b83b3fd294e8e9eb4614ec6214aa0] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=35.9 K 2024-11-15T04:40:48,921 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting a31a294c0c1b46768f1a08fdb8181b41, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731645647103 2024-11-15T04:40:48,921 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d1fe5427dce4e8aa84fdf6ef15b37d0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731645647103 2024-11-15T04:40:48,921 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 560c430b674544a28a10693374ce1c79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731645647139 2024-11-15T04:40:48,921 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 203fc2ecbabe4907abe8b2768eb51178, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731645647139 2024-11-15T04:40:48,921 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 974b83b3fd294e8e9eb4614ec6214aa0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731645648313 2024-11-15T04:40:48,921 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b77ad6462ec54bd49d74ed3d5f2bd589, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731645648312 2024-11-15T04:40:48,927 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:48,928 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/8638c33422214871b17841069fd844c8 is 50, key is test_row_0/A:col10/1731645648424/Put/seqid=0 2024-11-15T04:40:48,931 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:48,932 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/684e89857c0548f3902f218999deb1ba is 50, key is test_row_0/B:col10/1731645648424/Put/seqid=0 2024-11-15T04:40:48,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742115_1291 (size=12595) 2024-11-15T04:40:48,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742116_1292 (size=12595) 2024-11-15T04:40:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:49,097 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:49,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T04:40:49,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/7c838e8731924b9ab37ccd0a80aa8117 is 50, key is test_row_0/A:col10/1731645649096/Put/seqid=0 2024-11-15T04:40:49,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742117_1293 (size=16931) 2024-11-15T04:40:49,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/7c838e8731924b9ab37ccd0a80aa8117 2024-11-15T04:40:49,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/6e118c35949b441aa513e47628a86d55 is 50, key is test_row_0/B:col10/1731645649096/Put/seqid=0 2024-11-15T04:40:49,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742118_1294 (size=12151) 2024-11-15T04:40:49,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645709117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645709117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645709119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645709119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645709233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645709233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645709233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645709233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T04:40:49,303 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:49,303 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T04:40:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T04:40:49,305 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:49,306 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:49,306 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:49,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645709333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,336 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,340 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/8638c33422214871b17841069fd844c8 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8638c33422214871b17841069fd844c8 2024-11-15T04:40:49,344 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into 8638c33422214871b17841069fd844c8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:49,344 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:49,344 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=13, startTime=1731645648919; duration=0sec 2024-11-15T04:40:49,344 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:49,344 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:40:49,344 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:49,345 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:49,345 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/C is initiating minor compaction (all files) 2024-11-15T04:40:49,345 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/C in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:49,346 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/b3ca0075e8ea4d38b828971b10a47e49, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/fa27a161676640e5bb66b1d115808be4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2687a00026644a74907b8e7ded765cf6] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=35.9 K 2024-11-15T04:40:49,346 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ca0075e8ea4d38b828971b10a47e49, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731645647103 2024-11-15T04:40:49,346 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa27a161676640e5bb66b1d115808be4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731645647139 2024-11-15T04:40:49,346 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2687a00026644a74907b8e7ded765cf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731645648313 2024-11-15T04:40:49,353 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/684e89857c0548f3902f218999deb1ba as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/684e89857c0548f3902f218999deb1ba 2024-11-15T04:40:49,357 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#C#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:49,357 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d143656edd0d483ca63489eaf110a77b is 50, key is test_row_0/C:col10/1731645648424/Put/seqid=0 2024-11-15T04:40:49,357 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into 684e89857c0548f3902f218999deb1ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:49,357 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:49,357 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=13, startTime=1731645648919; duration=0sec 2024-11-15T04:40:49,358 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:49,358 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:40:49,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742119_1295 (size=12595) 2024-11-15T04:40:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T04:40:49,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645709442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645709442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645709442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645709442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T04:40:49,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:49,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:49,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:49,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/6e118c35949b441aa513e47628a86d55 2024-11-15T04:40:49,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2f77a3908880414eaee7a9280e580b4f is 50, key is test_row_0/C:col10/1731645649096/Put/seqid=0 2024-11-15T04:40:49,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742120_1296 (size=12151) 2024-11-15T04:40:49,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T04:40:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:49,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T04:40:49,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645709753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645709753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645709753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645709753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:49,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T04:40:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:49,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,770 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d143656edd0d483ca63489eaf110a77b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d143656edd0d483ca63489eaf110a77b 2024-11-15T04:40:49,777 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/C of e5b59d814d067d3a48ee3af2799f7c44 into d143656edd0d483ca63489eaf110a77b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:49,777 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:49,777 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/C, priority=13, startTime=1731645648919; duration=0sec 2024-11-15T04:40:49,777 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:49,777 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:40:49,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T04:40:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:49,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:49,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2f77a3908880414eaee7a9280e580b4f 2024-11-15T04:40:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T04:40:49,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/7c838e8731924b9ab37ccd0a80aa8117 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/7c838e8731924b9ab37ccd0a80aa8117 2024-11-15T04:40:49,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/7c838e8731924b9ab37ccd0a80aa8117, entries=250, sequenceid=208, filesize=16.5 K 2024-11-15T04:40:49,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/6e118c35949b441aa513e47628a86d55 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/6e118c35949b441aa513e47628a86d55 2024-11-15T04:40:49,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/6e118c35949b441aa513e47628a86d55, entries=150, sequenceid=208, filesize=11.9 K 2024-11-15T04:40:49,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/2f77a3908880414eaee7a9280e580b4f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2f77a3908880414eaee7a9280e580b4f 2024-11-15T04:40:49,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2f77a3908880414eaee7a9280e580b4f, entries=150, sequenceid=208, filesize=11.9 K 2024-11-15T04:40:49,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for e5b59d814d067d3a48ee3af2799f7c44 in 851ms, sequenceid=208, compaction requested=false 2024-11-15T04:40:49,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:50,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T04:40:50,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:50,069 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T04:40:50,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/678f35c6d8594f7fbf291d4f61cf51dc is 50, key is test_row_0/A:col10/1731645649115/Put/seqid=0 2024-11-15T04:40:50,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742121_1297 (size=12151) 2024-11-15T04:40:50,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:50,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:50,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645710288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645710289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645710292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645710292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,297 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645710402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645710402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645710402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645710402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T04:40:50,478 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/678f35c6d8594f7fbf291d4f61cf51dc 2024-11-15T04:40:50,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/490a7577b57446f896ee0cb2cf3ca679 is 50, key is test_row_0/B:col10/1731645649115/Put/seqid=0 2024-11-15T04:40:50,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742122_1298 (size=12151) 2024-11-15T04:40:50,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645710613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645710613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645710613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645710613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,637 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T04:40:50,637 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T04:40:50,899 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/490a7577b57446f896ee0cb2cf3ca679 2024-11-15T04:40:50,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/e81ae65c355549fe93b7c2f3dc92be19 is 50, key is test_row_0/C:col10/1731645649115/Put/seqid=0 2024-11-15T04:40:50,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742123_1299 (size=12151) 2024-11-15T04:40:50,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645710922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645710922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645710922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:50,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645710922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:50,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:51,310 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/e81ae65c355549fe93b7c2f3dc92be19 2024-11-15T04:40:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/678f35c6d8594f7fbf291d4f61cf51dc as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/678f35c6d8594f7fbf291d4f61cf51dc 2024-11-15T04:40:51,317 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/678f35c6d8594f7fbf291d4f61cf51dc, entries=150, sequenceid=224, filesize=11.9 K 2024-11-15T04:40:51,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/490a7577b57446f896ee0cb2cf3ca679 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/490a7577b57446f896ee0cb2cf3ca679 2024-11-15T04:40:51,321 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/490a7577b57446f896ee0cb2cf3ca679, entries=150, sequenceid=224, filesize=11.9 K 2024-11-15T04:40:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/e81ae65c355549fe93b7c2f3dc92be19 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e81ae65c355549fe93b7c2f3dc92be19 2024-11-15T04:40:51,325 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e81ae65c355549fe93b7c2f3dc92be19, entries=150, sequenceid=224, filesize=11.9 K 2024-11-15T04:40:51,326 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for e5b59d814d067d3a48ee3af2799f7c44 in 1257ms, sequenceid=224, compaction requested=true 2024-11-15T04:40:51,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:51,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:51,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T04:40:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T04:40:51,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T04:40:51,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0220 sec 2024-11-15T04:40:51,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.0270 sec 2024-11-15T04:40:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:51,345 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:51,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:40:51,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/b2504518a38c4c5ba273868f3a56ad3b is 50, key is test_row_0/A:col10/1731645651344/Put/seqid=0 2024-11-15T04:40:51,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742124_1300 (size=14541) 2024-11-15T04:40:51,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645711379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:51,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645711432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:51,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645711432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645711432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:51,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645711433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T04:40:51,453 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:51,454 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T04:40:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T04:40:51,456 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:51,456 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:51,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:51,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645711492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T04:40:51,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T04:40:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:51,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:51,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645711703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:51,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/b2504518a38c4c5ba273868f3a56ad3b 2024-11-15T04:40:51,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/c6e0026edd26417794c72a57dd3fac39 is 50, key is test_row_0/B:col10/1731645651344/Put/seqid=0 2024-11-15T04:40:51,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T04:40:51,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:51,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:51,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:51,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742125_1301 (size=12151) 2024-11-15T04:40:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T04:40:51,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T04:40:51,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:51,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:51,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:51,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:51,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645712012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:52,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T04:40:52,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:52,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T04:40:52,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/c6e0026edd26417794c72a57dd3fac39 2024-11-15T04:40:52,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/313676e69b8c4da9a451da24a86ffee5 is 50, key is test_row_0/C:col10/1731645651344/Put/seqid=0 2024-11-15T04:40:52,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742126_1302 (size=12151) 2024-11-15T04:40:52,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T04:40:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T04:40:52,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:52,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645712443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:52,445 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:52,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645712453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:52,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645712453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:52,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645712453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:52,455 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:52,455 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:52,456 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:52,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T04:40:52,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:52,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:52,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645712523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:52,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:52,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:52,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/313676e69b8c4da9a451da24a86ffee5 2024-11-15T04:40:52,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/b2504518a38c4c5ba273868f3a56ad3b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b2504518a38c4c5ba273868f3a56ad3b 2024-11-15T04:40:52,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b2504518a38c4c5ba273868f3a56ad3b, entries=200, sequenceid=249, filesize=14.2 K 2024-11-15T04:40:52,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/c6e0026edd26417794c72a57dd3fac39 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/c6e0026edd26417794c72a57dd3fac39 2024-11-15T04:40:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T04:40:52,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/c6e0026edd26417794c72a57dd3fac39, entries=150, sequenceid=249, filesize=11.9 K 2024-11-15T04:40:52,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/313676e69b8c4da9a451da24a86ffee5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/313676e69b8c4da9a451da24a86ffee5 2024-11-15T04:40:52,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/313676e69b8c4da9a451da24a86ffee5, entries=150, sequenceid=249, filesize=11.9 K 2024-11-15T04:40:52,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e5b59d814d067d3a48ee3af2799f7c44 in 1256ms, sequenceid=249, compaction requested=true 2024-11-15T04:40:52,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:52,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:52,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:52,601 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:52,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:52,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:52,601 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:52,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:52,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:52,602 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:52,602 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56218 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:52,602 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:40:52,602 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:40:52,602 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,602 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,602 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/684e89857c0548f3902f218999deb1ba, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/6e118c35949b441aa513e47628a86d55, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/490a7577b57446f896ee0cb2cf3ca679, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/c6e0026edd26417794c72a57dd3fac39] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=47.9 K 2024-11-15T04:40:52,602 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8638c33422214871b17841069fd844c8, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/7c838e8731924b9ab37ccd0a80aa8117, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/678f35c6d8594f7fbf291d4f61cf51dc, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b2504518a38c4c5ba273868f3a56ad3b] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=54.9 K 2024-11-15T04:40:52,603 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8638c33422214871b17841069fd844c8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731645648313 2024-11-15T04:40:52,603 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 684e89857c0548f3902f218999deb1ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731645648313 2024-11-15T04:40:52,603 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e118c35949b441aa513e47628a86d55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731645648456 2024-11-15T04:40:52,603 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c838e8731924b9ab37ccd0a80aa8117, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731645648456 2024-11-15T04:40:52,603 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 490a7577b57446f896ee0cb2cf3ca679, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731645649110 2024-11-15T04:40:52,603 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 678f35c6d8594f7fbf291d4f61cf51dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731645649110 2024-11-15T04:40:52,603 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e0026edd26417794c72a57dd3fac39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731645650290 2024-11-15T04:40:52,604 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2504518a38c4c5ba273868f3a56ad3b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731645650285 2024-11-15T04:40:52,611 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:52,612 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/9ed46ff418f2482588b586efa66a54d5 is 50, key is test_row_0/A:col10/1731645651344/Put/seqid=0 2024-11-15T04:40:52,614 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:52,615 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/eea4df3035d846eba4e373f306098707 is 50, key is test_row_0/B:col10/1731645651344/Put/seqid=0 2024-11-15T04:40:52,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742127_1303 (size=12731) 2024-11-15T04:40:52,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742128_1304 (size=12731) 2024-11-15T04:40:52,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T04:40:52,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:52,678 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:40:52,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/2f7a39ae97704dc59ceddbf6ff4c8602 is 50, key is test_row_0/A:col10/1731645651364/Put/seqid=0 2024-11-15T04:40:52,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742129_1305 (size=12201) 2024-11-15T04:40:53,025 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/9ed46ff418f2482588b586efa66a54d5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/9ed46ff418f2482588b586efa66a54d5 2024-11-15T04:40:53,069 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into 9ed46ff418f2482588b586efa66a54d5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:53,069 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:53,069 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=12, startTime=1731645652601; duration=0sec 2024-11-15T04:40:53,069 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:53,069 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:40:53,069 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:53,071 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:53,071 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/eea4df3035d846eba4e373f306098707 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/eea4df3035d846eba4e373f306098707 2024-11-15T04:40:53,071 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/C is initiating minor compaction (all files) 2024-11-15T04:40:53,071 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/C in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:53,071 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d143656edd0d483ca63489eaf110a77b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2f77a3908880414eaee7a9280e580b4f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e81ae65c355549fe93b7c2f3dc92be19, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/313676e69b8c4da9a451da24a86ffee5] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=47.9 K 2024-11-15T04:40:53,072 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d143656edd0d483ca63489eaf110a77b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731645648313 2024-11-15T04:40:53,072 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f77a3908880414eaee7a9280e580b4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731645648456 2024-11-15T04:40:53,072 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting e81ae65c355549fe93b7c2f3dc92be19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731645649110 2024-11-15T04:40:53,073 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 313676e69b8c4da9a451da24a86ffee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731645650290 2024-11-15T04:40:53,082 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into eea4df3035d846eba4e373f306098707(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:53,082 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:53,082 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=12, startTime=1731645652601; duration=0sec 2024-11-15T04:40:53,082 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:53,082 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:40:53,086 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/2f7a39ae97704dc59ceddbf6ff4c8602 2024-11-15T04:40:53,087 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:53,087 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/b51d12bfe8834520b5488e94e5ef4176 is 50, key is test_row_0/C:col10/1731645651344/Put/seqid=0 2024-11-15T04:40:53,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/1150813455194570b1ea70f1ab656c7d is 50, key is test_row_0/B:col10/1731645651364/Put/seqid=0 2024-11-15T04:40:53,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742130_1306 (size=12731) 2024-11-15T04:40:53,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742131_1307 (size=12201) 2024-11-15T04:40:53,136 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/1150813455194570b1ea70f1ab656c7d 2024-11-15T04:40:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d170e9b9fa904a56b11f89e0865cea01 is 50, key is test_row_0/C:col10/1731645651364/Put/seqid=0 2024-11-15T04:40:53,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742132_1308 (size=12201) 2024-11-15T04:40:53,153 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d170e9b9fa904a56b11f89e0865cea01 2024-11-15T04:40:53,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/2f7a39ae97704dc59ceddbf6ff4c8602 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/2f7a39ae97704dc59ceddbf6ff4c8602 2024-11-15T04:40:53,171 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/2f7a39ae97704dc59ceddbf6ff4c8602, entries=150, sequenceid=260, filesize=11.9 K 2024-11-15T04:40:53,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/1150813455194570b1ea70f1ab656c7d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/1150813455194570b1ea70f1ab656c7d 2024-11-15T04:40:53,176 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/1150813455194570b1ea70f1ab656c7d, entries=150, sequenceid=260, filesize=11.9 K 2024-11-15T04:40:53,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d170e9b9fa904a56b11f89e0865cea01 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d170e9b9fa904a56b11f89e0865cea01 2024-11-15T04:40:53,185 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d170e9b9fa904a56b11f89e0865cea01, entries=150, sequenceid=260, filesize=11.9 K 2024-11-15T04:40:53,186 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for e5b59d814d067d3a48ee3af2799f7c44 in 508ms, sequenceid=260, compaction requested=false 2024-11-15T04:40:53,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:53,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:53,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T04:40:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T04:40:53,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T04:40:53,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7340 sec 2024-11-15T04:40:53,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7410 sec 2024-11-15T04:40:53,516 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/b51d12bfe8834520b5488e94e5ef4176 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/b51d12bfe8834520b5488e94e5ef4176 2024-11-15T04:40:53,521 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/C of e5b59d814d067d3a48ee3af2799f7c44 into b51d12bfe8834520b5488e94e5ef4176(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:53,521 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:53,521 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/C, priority=12, startTime=1731645652601; duration=0sec 2024-11-15T04:40:53,521 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:53,521 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:40:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:53,570 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:53,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:40:53,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/c5381a838da44fef8adb52385c46b718 is 50, key is test_row_0/A:col10/1731645653568/Put/seqid=0 2024-11-15T04:40:53,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742133_1309 (size=14741) 2024-11-15T04:40:53,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T04:40:53,607 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:53,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T04:40:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T04:40:53,609 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:53,610 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:53,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:53,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645713692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:53,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T04:40:53,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T04:40:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:53,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:53,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:53,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645713803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:53,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T04:40:53,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:53,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:53,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:53,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:53,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T04:40:53,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/c5381a838da44fef8adb52385c46b718 2024-11-15T04:40:54,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/fa8bdd1f88394baf9cd4aebde4de511f is 50, key is test_row_0/B:col10/1731645653568/Put/seqid=0 2024-11-15T04:40:54,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645714013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:54,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742134_1310 (size=12301) 2024-11-15T04:40:54,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/fa8bdd1f88394baf9cd4aebde4de511f 2024-11-15T04:40:54,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d9d99a3b3f6747d69f1bef87056ed9c4 is 50, key is test_row_0/C:col10/1731645653568/Put/seqid=0 2024-11-15T04:40:54,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T04:40:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742135_1311 (size=12301) 2024-11-15T04:40:54,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T04:40:54,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:54,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T04:40:54,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645714323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:54,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T04:40:54,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:54,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645714462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:54,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645714462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:54,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645714463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:54,465 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:54,465 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:54,467 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:54,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645714473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:54,475 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:54,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:54,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d9d99a3b3f6747d69f1bef87056ed9c4 2024-11-15T04:40:54,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/c5381a838da44fef8adb52385c46b718 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c5381a838da44fef8adb52385c46b718 2024-11-15T04:40:54,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c5381a838da44fef8adb52385c46b718, entries=200, sequenceid=274, filesize=14.4 K 2024-11-15T04:40:54,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/fa8bdd1f88394baf9cd4aebde4de511f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/fa8bdd1f88394baf9cd4aebde4de511f 2024-11-15T04:40:54,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/fa8bdd1f88394baf9cd4aebde4de511f, entries=150, sequenceid=274, filesize=12.0 K 2024-11-15T04:40:54,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d9d99a3b3f6747d69f1bef87056ed9c4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d9d99a3b3f6747d69f1bef87056ed9c4 2024-11-15T04:40:54,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d9d99a3b3f6747d69f1bef87056ed9c4, entries=150, sequenceid=274, filesize=12.0 K 2024-11-15T04:40:54,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e5b59d814d067d3a48ee3af2799f7c44 in 937ms, sequenceid=274, compaction requested=true 2024-11-15T04:40:54,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:54,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:54,508 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:54,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:54,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:54,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:54,508 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:54,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:54,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:54,508 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:54,508 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39673 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:54,508 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:40:54,508 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:40:54,508 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,508 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,509 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/9ed46ff418f2482588b586efa66a54d5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/2f7a39ae97704dc59ceddbf6ff4c8602, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c5381a838da44fef8adb52385c46b718] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=38.7 K 2024-11-15T04:40:54,509 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/eea4df3035d846eba4e373f306098707, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/1150813455194570b1ea70f1ab656c7d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/fa8bdd1f88394baf9cd4aebde4de511f] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=36.4 K 2024-11-15T04:40:54,509 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting eea4df3035d846eba4e373f306098707, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731645650290 2024-11-15T04:40:54,509 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ed46ff418f2482588b586efa66a54d5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731645650290 2024-11-15T04:40:54,509 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f7a39ae97704dc59ceddbf6ff4c8602, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731645651348 2024-11-15T04:40:54,509 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5381a838da44fef8adb52385c46b718, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731645653555 2024-11-15T04:40:54,510 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 1150813455194570b1ea70f1ab656c7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731645651348 2024-11-15T04:40:54,510 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting fa8bdd1f88394baf9cd4aebde4de511f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731645653555 2024-11-15T04:40:54,515 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:54,515 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:54,515 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/5d046a8febc34cce919f8612d90d0aac is 50, key is test_row_0/B:col10/1731645653568/Put/seqid=0 2024-11-15T04:40:54,516 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/10490c39a0e748aa8a4712fb8df4ad40 is 50, key is test_row_0/A:col10/1731645653568/Put/seqid=0 2024-11-15T04:40:54,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742136_1312 (size=12983) 2024-11-15T04:40:54,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742137_1313 (size=12983) 2024-11-15T04:40:54,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T04:40:54,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,534 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:40:54,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/5eb165be0f7345f9b375d4826ba088c4 is 50, key is test_row_0/A:col10/1731645653677/Put/seqid=0 2024-11-15T04:40:54,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742138_1314 (size=12301) 2024-11-15T04:40:54,547 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/5eb165be0f7345f9b375d4826ba088c4 2024-11-15T04:40:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/61b1ef86cad3485ebf939d7757a258f7 is 50, key is test_row_0/B:col10/1731645653677/Put/seqid=0 2024-11-15T04:40:54,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742139_1315 (size=12301) 2024-11-15T04:40:54,560 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/61b1ef86cad3485ebf939d7757a258f7 2024-11-15T04:40:54,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/ab658f0c454b4e45a244cd4168a7791a is 50, key is test_row_0/C:col10/1731645653677/Put/seqid=0 2024-11-15T04:40:54,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742140_1316 (size=12301) 2024-11-15T04:40:54,572 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/ab658f0c454b4e45a244cd4168a7791a 2024-11-15T04:40:54,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/5eb165be0f7345f9b375d4826ba088c4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/5eb165be0f7345f9b375d4826ba088c4 2024-11-15T04:40:54,579 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/5eb165be0f7345f9b375d4826ba088c4, entries=150, sequenceid=299, filesize=12.0 K 2024-11-15T04:40:54,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/61b1ef86cad3485ebf939d7757a258f7 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/61b1ef86cad3485ebf939d7757a258f7 2024-11-15T04:40:54,583 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/61b1ef86cad3485ebf939d7757a258f7, entries=150, sequenceid=299, filesize=12.0 K 2024-11-15T04:40:54,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/ab658f0c454b4e45a244cd4168a7791a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ab658f0c454b4e45a244cd4168a7791a 2024-11-15T04:40:54,588 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ab658f0c454b4e45a244cd4168a7791a, entries=150, sequenceid=299, filesize=12.0 K 2024-11-15T04:40:54,589 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for e5b59d814d067d3a48ee3af2799f7c44 in 55ms, sequenceid=299, compaction requested=true 2024-11-15T04:40:54,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:54,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T04:40:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T04:40:54,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T04:40:54,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 981 msec 2024-11-15T04:40:54,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 987 msec 2024-11-15T04:40:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T04:40:54,743 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:54,744 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-15T04:40:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T04:40:54,746 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:54,747 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:54,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T04:40:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:54,861 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:54,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:40:54,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/14006e3f391741349ea12d304bc17fab is 50, key is test_row_0/A:col10/1731645654860/Put/seqid=0 2024-11-15T04:40:54,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742141_1317 (size=14741) 2024-11-15T04:40:54,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/14006e3f391741349ea12d304bc17fab 2024-11-15T04:40:54,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/abc94552014a43998473bedf7f959f65 is 50, key is test_row_0/B:col10/1731645654860/Put/seqid=0 2024-11-15T04:40:54,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T04:40:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:54,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742142_1318 (size=12301) 2024-11-15T04:40:54,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/abc94552014a43998473bedf7f959f65 2024-11-15T04:40:54,931 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/10490c39a0e748aa8a4712fb8df4ad40 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/10490c39a0e748aa8a4712fb8df4ad40 2024-11-15T04:40:54,932 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/5d046a8febc34cce919f8612d90d0aac as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/5d046a8febc34cce919f8612d90d0aac 2024-11-15T04:40:54,939 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into 10490c39a0e748aa8a4712fb8df4ad40(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:54,939 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:54,939 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=13, startTime=1731645654507; duration=0sec 2024-11-15T04:40:54,940 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:54,940 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:40:54,940 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:54,943 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into 5d046a8febc34cce919f8612d90d0aac(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:54,943 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:54,943 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=13, startTime=1731645654508; duration=0sec 2024-11-15T04:40:54,943 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49534 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:54,943 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/C is initiating minor compaction (all files) 2024-11-15T04:40:54,943 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:54,943 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:40:54,943 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/C in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:54,943 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/b51d12bfe8834520b5488e94e5ef4176, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d170e9b9fa904a56b11f89e0865cea01, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d9d99a3b3f6747d69f1bef87056ed9c4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ab658f0c454b4e45a244cd4168a7791a] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=48.4 K 2024-11-15T04:40:54,944 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51d12bfe8834520b5488e94e5ef4176, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731645650290 2024-11-15T04:40:54,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/9ac6509fe1784010bee3ed54aa09b466 is 50, key is test_row_0/C:col10/1731645654860/Put/seqid=0 2024-11-15T04:40:54,945 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d170e9b9fa904a56b11f89e0865cea01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731645651348 2024-11-15T04:40:54,946 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9d99a3b3f6747d69f1bef87056ed9c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731645653555 2024-11-15T04:40:54,947 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab658f0c454b4e45a244cd4168a7791a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731645653677 2024-11-15T04:40:54,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742143_1319 (size=12301) 2024-11-15T04:40:54,977 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:54,978 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/27c48efcc72943599456216da03fdfde is 50, key is test_row_0/C:col10/1731645653677/Put/seqid=0 2024-11-15T04:40:55,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742144_1320 (size=13017) 2024-11-15T04:40:55,008 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/27c48efcc72943599456216da03fdfde as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/27c48efcc72943599456216da03fdfde 2024-11-15T04:40:55,013 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/C of e5b59d814d067d3a48ee3af2799f7c44 into 27c48efcc72943599456216da03fdfde(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:55,013 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:55,013 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/C, priority=12, startTime=1731645654508; duration=0sec 2024-11-15T04:40:55,013 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:55,013 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:40:55,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645715016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:55,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T04:40:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:55,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T04:40:55,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645715133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:55,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T04:40:55,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:55,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:55,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:55,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:55,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:55,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645715343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:55,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:55,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/9ac6509fe1784010bee3ed54aa09b466 2024-11-15T04:40:55,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T04:40:55,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:55,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:55,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:55,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:55,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:55,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:55,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/14006e3f391741349ea12d304bc17fab as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/14006e3f391741349ea12d304bc17fab 2024-11-15T04:40:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T04:40:55,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/14006e3f391741349ea12d304bc17fab, entries=200, sequenceid=310, filesize=14.4 K 2024-11-15T04:40:55,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/abc94552014a43998473bedf7f959f65 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/abc94552014a43998473bedf7f959f65 2024-11-15T04:40:55,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/abc94552014a43998473bedf7f959f65, entries=150, sequenceid=310, filesize=12.0 K 2024-11-15T04:40:55,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/9ac6509fe1784010bee3ed54aa09b466 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9ac6509fe1784010bee3ed54aa09b466 2024-11-15T04:40:55,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9ac6509fe1784010bee3ed54aa09b466, entries=150, sequenceid=310, filesize=12.0 K 2024-11-15T04:40:55,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e5b59d814d067d3a48ee3af2799f7c44 in 528ms, sequenceid=310, compaction requested=true 2024-11-15T04:40:55,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:55,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:55,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:55,389 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:55,390 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:55,391 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:55,391 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:55,391 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:40:55,391 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:40:55,391 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:55,391 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:55,391 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/10490c39a0e748aa8a4712fb8df4ad40, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/5eb165be0f7345f9b375d4826ba088c4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/14006e3f391741349ea12d304bc17fab] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=39.1 K 2024-11-15T04:40:55,391 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/5d046a8febc34cce919f8612d90d0aac, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/61b1ef86cad3485ebf939d7757a258f7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/abc94552014a43998473bedf7f959f65] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=36.7 K 2024-11-15T04:40:55,392 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10490c39a0e748aa8a4712fb8df4ad40, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731645653555 2024-11-15T04:40:55,392 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d046a8febc34cce919f8612d90d0aac, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731645653555 2024-11-15T04:40:55,392 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eb165be0f7345f9b375d4826ba088c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731645653677 2024-11-15T04:40:55,392 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 61b1ef86cad3485ebf939d7757a258f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731645653677 2024-11-15T04:40:55,393 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14006e3f391741349ea12d304bc17fab, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731645654842 2024-11-15T04:40:55,393 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting abc94552014a43998473bedf7f959f65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731645654842 2024-11-15T04:40:55,401 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:55,402 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/78971e822078448080d151e4ac363f96 is 50, key is test_row_0/B:col10/1731645654860/Put/seqid=0 2024-11-15T04:40:55,402 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:55,402 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/c2a7cd18897b489788334b0967ff95b2 is 50, key is test_row_0/A:col10/1731645654860/Put/seqid=0 2024-11-15T04:40:55,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742145_1321 (size=13085) 2024-11-15T04:40:55,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742146_1322 (size=13085) 2024-11-15T04:40:55,454 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/c2a7cd18897b489788334b0967ff95b2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c2a7cd18897b489788334b0967ff95b2 2024-11-15T04:40:55,458 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/78971e822078448080d151e4ac363f96 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/78971e822078448080d151e4ac363f96 2024-11-15T04:40:55,462 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into c2a7cd18897b489788334b0967ff95b2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:55,462 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:55,462 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=13, startTime=1731645655389; duration=0sec 2024-11-15T04:40:55,462 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:55,462 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:40:55,462 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T04:40:55,464 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T04:40:55,464 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T04:40:55,464 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. because compaction request was cancelled 2024-11-15T04:40:55,464 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:40:55,469 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into 78971e822078448080d151e4ac363f96(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:55,469 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:55,469 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=13, startTime=1731645655389; duration=0sec 2024-11-15T04:40:55,470 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:55,470 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:40:55,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T04:40:55,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:55,516 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:40:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d1ba7ce027b44fb5923d0a722e7fcd52 is 50, key is test_row_0/A:col10/1731645655014/Put/seqid=0 2024-11-15T04:40:55,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742147_1323 (size=12301) 2024-11-15T04:40:55,597 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d1ba7ce027b44fb5923d0a722e7fcd52 2024-11-15T04:40:55,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/83b092774e2546b9a109e9b9b40ae496 is 50, key is test_row_0/B:col10/1731645655014/Put/seqid=0 2024-11-15T04:40:55,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:55,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742148_1324 (size=12301) 2024-11-15T04:40:55,665 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/83b092774e2546b9a109e9b9b40ae496 2024-11-15T04:40:55,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/a9a71a6c19814c1dbb66084ee6e20e8c is 50, key is test_row_0/C:col10/1731645655014/Put/seqid=0 2024-11-15T04:40:55,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645715734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:55,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:55,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742149_1325 (size=12301) 2024-11-15T04:40:55,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645715843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T04:40:56,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:56,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645716053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:56,170 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/a9a71a6c19814c1dbb66084ee6e20e8c 2024-11-15T04:40:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d1ba7ce027b44fb5923d0a722e7fcd52 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d1ba7ce027b44fb5923d0a722e7fcd52 2024-11-15T04:40:56,190 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d1ba7ce027b44fb5923d0a722e7fcd52, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T04:40:56,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/83b092774e2546b9a109e9b9b40ae496 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83b092774e2546b9a109e9b9b40ae496 2024-11-15T04:40:56,199 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83b092774e2546b9a109e9b9b40ae496, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T04:40:56,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/a9a71a6c19814c1dbb66084ee6e20e8c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9a71a6c19814c1dbb66084ee6e20e8c 2024-11-15T04:40:56,211 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9a71a6c19814c1dbb66084ee6e20e8c, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T04:40:56,216 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e5b59d814d067d3a48ee3af2799f7c44 in 700ms, sequenceid=340, compaction requested=true 2024-11-15T04:40:56,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:56,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:56,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-15T04:40:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-15T04:40:56,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T04:40:56,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4700 sec 2024-11-15T04:40:56,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4770 sec 2024-11-15T04:40:56,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:56,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:56,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T04:40:56,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d852c634ed7d44d39007a2f03d405daf is 50, key is test_row_1/A:col10/1731645655725/Put/seqid=0 2024-11-15T04:40:56,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742150_1326 (size=12297) 2024-11-15T04:40:56,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645716497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:56,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645716603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:56,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645716812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:56,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d852c634ed7d44d39007a2f03d405daf 2024-11-15T04:40:56,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/43aadd2493bf456f92b562404870c7d5 is 50, key is test_row_1/B:col10/1731645655725/Put/seqid=0 2024-11-15T04:40:56,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742151_1327 (size=9857) 2024-11-15T04:40:56,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/43aadd2493bf456f92b562404870c7d5 2024-11-15T04:40:56,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/ec0b7980813041109c9f3c8ccaa731fe is 50, key is test_row_1/C:col10/1731645655725/Put/seqid=0 2024-11-15T04:40:56,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742152_1328 (size=9857) 2024-11-15T04:40:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T04:40:56,893 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:56,893 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-15T04:40:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T04:40:56,895 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:56,896 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:56,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T04:40:57,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T04:40:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:57,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:57,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:57,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645717122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:57,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T04:40:57,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:57,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:57,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:57,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:57,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:40:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T04:40:57,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/ec0b7980813041109c9f3c8ccaa731fe 2024-11-15T04:40:57,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d852c634ed7d44d39007a2f03d405daf as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d852c634ed7d44d39007a2f03d405daf 2024-11-15T04:40:57,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d852c634ed7d44d39007a2f03d405daf, entries=150, sequenceid=352, filesize=12.0 K 2024-11-15T04:40:57,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/43aadd2493bf456f92b562404870c7d5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/43aadd2493bf456f92b562404870c7d5 2024-11-15T04:40:57,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/43aadd2493bf456f92b562404870c7d5, entries=100, sequenceid=352, filesize=9.6 K 2024-11-15T04:40:57,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/ec0b7980813041109c9f3c8ccaa731fe as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ec0b7980813041109c9f3c8ccaa731fe 2024-11-15T04:40:57,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ec0b7980813041109c9f3c8ccaa731fe, entries=100, sequenceid=352, filesize=9.6 K 2024-11-15T04:40:57,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e5b59d814d067d3a48ee3af2799f7c44 in 951ms, sequenceid=352, compaction requested=true 2024-11-15T04:40:57,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:57,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:57,315 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:57,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:57,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:57,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:57,315 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:57,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:57,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:57,316 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:57,316 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:57,316 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:40:57,316 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:40:57,316 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:57,316 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:57,317 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/78971e822078448080d151e4ac363f96, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83b092774e2546b9a109e9b9b40ae496, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/43aadd2493bf456f92b562404870c7d5] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=34.4 K 2024-11-15T04:40:57,317 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c2a7cd18897b489788334b0967ff95b2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d1ba7ce027b44fb5923d0a722e7fcd52, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d852c634ed7d44d39007a2f03d405daf] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=36.8 K 2024-11-15T04:40:57,317 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a7cd18897b489788334b0967ff95b2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731645654842 2024-11-15T04:40:57,317 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 78971e822078448080d151e4ac363f96, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731645654842 2024-11-15T04:40:57,317 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1ba7ce027b44fb5923d0a722e7fcd52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731645654945 2024-11-15T04:40:57,317 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 83b092774e2546b9a109e9b9b40ae496, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731645654945 2024-11-15T04:40:57,318 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d852c634ed7d44d39007a2f03d405daf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731645655725 2024-11-15T04:40:57,318 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 43aadd2493bf456f92b562404870c7d5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731645655725 2024-11-15T04:40:57,329 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:57,329 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/be4e3c1a23d740e7a2b8b3673521a162 is 50, key is test_row_0/B:col10/1731645655014/Put/seqid=0 2024-11-15T04:40:57,332 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:57,332 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/96e77f6cc82048fe966d7c44a487c654 is 50, key is test_row_0/A:col10/1731645655014/Put/seqid=0 2024-11-15T04:40:57,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742153_1329 (size=13187) 2024-11-15T04:40:57,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742154_1330 (size=13187) 2024-11-15T04:40:57,349 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/96e77f6cc82048fe966d7c44a487c654 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/96e77f6cc82048fe966d7c44a487c654 2024-11-15T04:40:57,353 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into 96e77f6cc82048fe966d7c44a487c654(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:57,353 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:57,353 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=13, startTime=1731645657315; duration=0sec 2024-11-15T04:40:57,353 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:57,353 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:40:57,353 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:40:57,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T04:40:57,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:57,354 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T04:40:57,355 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:40:57,355 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/C is initiating minor compaction (all files) 2024-11-15T04:40:57,355 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/C in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:57,355 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/27c48efcc72943599456216da03fdfde, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9ac6509fe1784010bee3ed54aa09b466, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9a71a6c19814c1dbb66084ee6e20e8c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ec0b7980813041109c9f3c8ccaa731fe] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=46.4 K 2024-11-15T04:40:57,356 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27c48efcc72943599456216da03fdfde, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731645653677 2024-11-15T04:40:57,356 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ac6509fe1784010bee3ed54aa09b466, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731645654842 2024-11-15T04:40:57,356 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9a71a6c19814c1dbb66084ee6e20e8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731645654945 2024-11-15T04:40:57,356 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec0b7980813041109c9f3c8ccaa731fe, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731645655725 2024-11-15T04:40:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d73a2ba625714f27963a0f9cd343d1ba is 50, key is test_row_0/A:col10/1731645656489/Put/seqid=0 2024-11-15T04:40:57,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742155_1331 (size=12301) 2024-11-15T04:40:57,366 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#C#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:57,366 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/9d03e2a29e194d47b7ea21683a707dbb is 50, key is test_row_0/C:col10/1731645655014/Put/seqid=0 2024-11-15T04:40:57,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742156_1332 (size=13153) 2024-11-15T04:40:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T04:40:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:57,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:57,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645717665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:57,748 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/be4e3c1a23d740e7a2b8b3673521a162 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/be4e3c1a23d740e7a2b8b3673521a162 2024-11-15T04:40:57,754 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into be4e3c1a23d740e7a2b8b3673521a162(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:57,754 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:57,754 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=13, startTime=1731645657315; duration=0sec 2024-11-15T04:40:57,754 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:57,755 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:40:57,764 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d73a2ba625714f27963a0f9cd343d1ba 2024-11-15T04:40:57,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:57,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645717773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:57,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:57,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:57,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:57,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/72e35c48941d48e490dc37bc380aacf0 is 50, key is test_row_0/B:col10/1731645656489/Put/seqid=0 2024-11-15T04:40:57,782 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/9d03e2a29e194d47b7ea21683a707dbb as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9d03e2a29e194d47b7ea21683a707dbb 2024-11-15T04:40:57,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742157_1333 (size=12301) 2024-11-15T04:40:57,787 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/C of e5b59d814d067d3a48ee3af2799f7c44 into 9d03e2a29e194d47b7ea21683a707dbb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:57,787 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:57,787 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/C, priority=12, startTime=1731645657315; duration=0sec 2024-11-15T04:40:57,787 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:57,787 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:40:57,789 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/72e35c48941d48e490dc37bc380aacf0 2024-11-15T04:40:57,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d938455e058346e5af3c899609c27d0a is 50, key is test_row_0/C:col10/1731645656489/Put/seqid=0 2024-11-15T04:40:57,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742158_1334 (size=12301) 2024-11-15T04:40:57,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645717983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T04:40:58,218 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d938455e058346e5af3c899609c27d0a 2024-11-15T04:40:58,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/d73a2ba625714f27963a0f9cd343d1ba as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d73a2ba625714f27963a0f9cd343d1ba 2024-11-15T04:40:58,225 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d73a2ba625714f27963a0f9cd343d1ba, entries=150, sequenceid=377, filesize=12.0 K 2024-11-15T04:40:58,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/72e35c48941d48e490dc37bc380aacf0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/72e35c48941d48e490dc37bc380aacf0 2024-11-15T04:40:58,231 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/72e35c48941d48e490dc37bc380aacf0, entries=150, sequenceid=377, filesize=12.0 K 2024-11-15T04:40:58,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/d938455e058346e5af3c899609c27d0a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d938455e058346e5af3c899609c27d0a 2024-11-15T04:40:58,237 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d938455e058346e5af3c899609c27d0a, entries=150, sequenceid=377, filesize=12.0 K 2024-11-15T04:40:58,238 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e5b59d814d067d3a48ee3af2799f7c44 in 884ms, sequenceid=377, compaction requested=false 2024-11-15T04:40:58,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:58,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:58,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-15T04:40:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-15T04:40:58,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T04:40:58,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3440 sec 2024-11-15T04:40:58,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.3500 sec 2024-11-15T04:40:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:58,296 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:40:58,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T04:40:58,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/43f2ae9581924090a35131ec058da563 is 50, key is test_row_0/A:col10/1731645657663/Put/seqid=0 2024-11-15T04:40:58,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742159_1335 (size=14741) 2024-11-15T04:40:58,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645718372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,377 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:58,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54304 deadline: 1731645718483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:58,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54282 deadline: 1731645718483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:58,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:58,485 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54266 deadline: 1731645718483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:58,486 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:58,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645718483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:58,486 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:58,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:58,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54306 deadline: 1731645718502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:58,505 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in e5b59d814d067d3a48ee3af2799f7c44 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:40:58,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:58,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645718693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:58,694 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:58,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/43f2ae9581924090a35131ec058da563 2024-11-15T04:40:58,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/ec78b22048e741afb23632c931f64b44 is 50, key is test_row_0/B:col10/1731645657663/Put/seqid=0 2024-11-15T04:40:58,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742160_1336 (size=12301) 2024-11-15T04:40:59,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645719003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T04:40:59,043 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:40:59,043 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:40:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-15T04:40:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T04:40:59,045 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:40:59,045 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:40:59,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:40:59,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/ec78b22048e741afb23632c931f64b44 2024-11-15T04:40:59,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/a9b08f3631424254a6d10ccaef583922 is 50, key is test_row_0/C:col10/1731645657663/Put/seqid=0 2024-11-15T04:40:59,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742161_1337 (size=12301) 2024-11-15T04:40:59,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/a9b08f3631424254a6d10ccaef583922 2024-11-15T04:40:59,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/43f2ae9581924090a35131ec058da563 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/43f2ae9581924090a35131ec058da563 2024-11-15T04:40:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T04:40:59,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/43f2ae9581924090a35131ec058da563, entries=200, sequenceid=392, filesize=14.4 K 2024-11-15T04:40:59,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/ec78b22048e741afb23632c931f64b44 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/ec78b22048e741afb23632c931f64b44 2024-11-15T04:40:59,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/ec78b22048e741afb23632c931f64b44, entries=150, sequenceid=392, filesize=12.0 K 2024-11-15T04:40:59,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/a9b08f3631424254a6d10ccaef583922 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9b08f3631424254a6d10ccaef583922 2024-11-15T04:40:59,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9b08f3631424254a6d10ccaef583922, entries=150, sequenceid=392, filesize=12.0 K 2024-11-15T04:40:59,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e5b59d814d067d3a48ee3af2799f7c44 in 873ms, sequenceid=392, compaction requested=true 2024-11-15T04:40:59,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:59,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:40:59,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:59,170 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:59,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:40:59,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:59,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:40:59,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T04:40:59,170 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:59,170 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:59,170 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:59,170 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:40:59,170 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:40:59,170 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:59,170 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:59,170 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/be4e3c1a23d740e7a2b8b3673521a162, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/72e35c48941d48e490dc37bc380aacf0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/ec78b22048e741afb23632c931f64b44] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=36.9 K 2024-11-15T04:40:59,170 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/96e77f6cc82048fe966d7c44a487c654, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d73a2ba625714f27963a0f9cd343d1ba, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/43f2ae9581924090a35131ec058da563] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=39.3 K 2024-11-15T04:40:59,171 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96e77f6cc82048fe966d7c44a487c654, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731645655014 2024-11-15T04:40:59,171 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d73a2ba625714f27963a0f9cd343d1ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731645656472 2024-11-15T04:40:59,171 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting be4e3c1a23d740e7a2b8b3673521a162, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731645655014 2024-11-15T04:40:59,172 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43f2ae9581924090a35131ec058da563, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731645657652 2024-11-15T04:40:59,172 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 72e35c48941d48e490dc37bc380aacf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731645656472 2024-11-15T04:40:59,172 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting ec78b22048e741afb23632c931f64b44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731645657652 2024-11-15T04:40:59,194 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:59,194 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/cf12ccf9c67b4124a14bbe606bfe6bc6 is 50, key is test_row_0/B:col10/1731645657663/Put/seqid=0 2024-11-15T04:40:59,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T04:40:59,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:59,198 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:59,198 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T04:40:59,199 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/86a1d45c728e4cffb9cb8898aede09a2 is 50, key is test_row_0/A:col10/1731645657663/Put/seqid=0 2024-11-15T04:40:59,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/84181974f2a147beae6c9a1f7089b681 is 50, key is test_row_0/A:col10/1731645658371/Put/seqid=0 2024-11-15T04:40:59,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742162_1338 (size=13289) 2024-11-15T04:40:59,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742163_1339 (size=13289) 2024-11-15T04:40:59,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742164_1340 (size=12301) 2024-11-15T04:40:59,217 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/84181974f2a147beae6c9a1f7089b681 2024-11-15T04:40:59,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/3ef38cd785e74c8cb16e827891340144 is 50, key is test_row_0/B:col10/1731645658371/Put/seqid=0 2024-11-15T04:40:59,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742165_1341 (size=12301) 2024-11-15T04:40:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T04:40:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:40:59,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:40:59,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645719548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:59,618 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/cf12ccf9c67b4124a14bbe606bfe6bc6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/cf12ccf9c67b4124a14bbe606bfe6bc6 2024-11-15T04:40:59,619 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/86a1d45c728e4cffb9cb8898aede09a2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/86a1d45c728e4cffb9cb8898aede09a2 2024-11-15T04:40:59,623 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into cf12ccf9c67b4124a14bbe606bfe6bc6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:59,623 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into 86a1d45c728e4cffb9cb8898aede09a2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:59,623 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:59,623 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:59,623 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=13, startTime=1731645659169; duration=0sec 2024-11-15T04:40:59,623 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=13, startTime=1731645659170; duration=0sec 2024-11-15T04:40:59,623 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:40:59,623 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:40:59,623 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:59,623 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:40:59,624 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:40:59,625 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:40:59,625 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/C is initiating minor compaction (all files) 2024-11-15T04:40:59,625 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/C in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:40:59,625 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9d03e2a29e194d47b7ea21683a707dbb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d938455e058346e5af3c899609c27d0a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9b08f3631424254a6d10ccaef583922] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=36.9 K 2024-11-15T04:40:59,625 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d03e2a29e194d47b7ea21683a707dbb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731645655014 2024-11-15T04:40:59,625 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d938455e058346e5af3c899609c27d0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731645656472 2024-11-15T04:40:59,626 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9b08f3631424254a6d10ccaef583922, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731645657652 2024-11-15T04:40:59,629 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/3ef38cd785e74c8cb16e827891340144 2024-11-15T04:40:59,640 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#C#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:40:59,640 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/7ce2b32bb2644dc18d73533f6a3597b1 is 50, key is test_row_0/C:col10/1731645657663/Put/seqid=0 2024-11-15T04:40:59,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/be35e01714df4c1baeecbf2410f2672e is 50, key is test_row_0/C:col10/1731645658371/Put/seqid=0 2024-11-15T04:40:59,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742166_1342 (size=12301) 2024-11-15T04:40:59,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:59,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645719662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:59,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:40:59,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742167_1343 (size=13255) 2024-11-15T04:40:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T04:40:59,678 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/7ce2b32bb2644dc18d73533f6a3597b1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/7ce2b32bb2644dc18d73533f6a3597b1 2024-11-15T04:40:59,694 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/C of e5b59d814d067d3a48ee3af2799f7c44 into 7ce2b32bb2644dc18d73533f6a3597b1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:40:59,694 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:40:59,694 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/C, priority=13, startTime=1731645659170; duration=0sec 2024-11-15T04:40:59,695 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:40:59,695 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:40:59,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:40:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645719872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:40:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:40:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:00,064 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/be35e01714df4c1baeecbf2410f2672e 2024-11-15T04:41:00,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/84181974f2a147beae6c9a1f7089b681 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/84181974f2a147beae6c9a1f7089b681 2024-11-15T04:41:00,078 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/84181974f2a147beae6c9a1f7089b681, entries=150, sequenceid=415, filesize=12.0 K 2024-11-15T04:41:00,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/3ef38cd785e74c8cb16e827891340144 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/3ef38cd785e74c8cb16e827891340144 2024-11-15T04:41:00,084 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/3ef38cd785e74c8cb16e827891340144, entries=150, sequenceid=415, filesize=12.0 K 2024-11-15T04:41:00,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/be35e01714df4c1baeecbf2410f2672e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/be35e01714df4c1baeecbf2410f2672e 2024-11-15T04:41:00,092 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/be35e01714df4c1baeecbf2410f2672e, entries=150, sequenceid=415, filesize=12.0 K 2024-11-15T04:41:00,093 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e5b59d814d067d3a48ee3af2799f7c44 in 894ms, sequenceid=415, compaction requested=false 2024-11-15T04:41:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:41:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-15T04:41:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-15T04:41:00,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-15T04:41:00,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0490 sec 2024-11-15T04:41:00,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0550 sec 2024-11-15T04:41:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T04:41:00,183 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:00,183 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-15T04:41:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:41:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T04:41:00,185 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:00,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T04:41:00,185 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:00,186 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:00,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:00,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/53dd76b88e4349ffb85cc3bdcb88c8aa is 50, key is test_row_0/A:col10/1731645659530/Put/seqid=0 2024-11-15T04:41:00,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742168_1344 (size=12301) 2024-11-15T04:41:00,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645720260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T04:41:00,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T04:41:00,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:41:00,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645720372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:00,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:00,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T04:41:00,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:41:00,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T04:41:00,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645720582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:00,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:00,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:00,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:00,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/53dd76b88e4349ffb85cc3bdcb88c8aa 2024-11-15T04:41:00,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/0feaafdd2b58407b97be8facb431cde3 is 50, key is test_row_0/B:col10/1731645659530/Put/seqid=0 2024-11-15T04:41:00,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742169_1345 (size=12301) 2024-11-15T04:41:00,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/0feaafdd2b58407b97be8facb431cde3 2024-11-15T04:41:00,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/9e81d346824847a989ac9d25bebdfc31 is 50, key is test_row_0/C:col10/1731645659530/Put/seqid=0 2024-11-15T04:41:00,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742170_1346 (size=12301) 2024-11-15T04:41:00,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T04:41:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:41:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T04:41:00,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:41:00,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T04:41:00,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645720893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:00,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:00,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T04:41:00,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:41:00,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:00,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:01,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/9e81d346824847a989ac9d25bebdfc31 2024-11-15T04:41:01,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/53dd76b88e4349ffb85cc3bdcb88c8aa as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/53dd76b88e4349ffb85cc3bdcb88c8aa 2024-11-15T04:41:01,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/53dd76b88e4349ffb85cc3bdcb88c8aa, entries=150, sequenceid=432, filesize=12.0 K 2024-11-15T04:41:01,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/0feaafdd2b58407b97be8facb431cde3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/0feaafdd2b58407b97be8facb431cde3 2024-11-15T04:41:01,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/0feaafdd2b58407b97be8facb431cde3, entries=150, sequenceid=432, filesize=12.0 K 2024-11-15T04:41:01,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/9e81d346824847a989ac9d25bebdfc31 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9e81d346824847a989ac9d25bebdfc31 2024-11-15T04:41:01,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9e81d346824847a989ac9d25bebdfc31, entries=150, sequenceid=432, filesize=12.0 K 2024-11-15T04:41:01,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for e5b59d814d067d3a48ee3af2799f7c44 in 841ms, sequenceid=432, compaction requested=true 2024-11-15T04:41:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:41:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:01,026 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:01,026 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:01,027 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:01,027 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:41:01,027 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:01,028 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/cf12ccf9c67b4124a14bbe606bfe6bc6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/3ef38cd785e74c8cb16e827891340144, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/0feaafdd2b58407b97be8facb431cde3] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=37.0 K 2024-11-15T04:41:01,028 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:01,028 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:41:01,028 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:01,028 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/86a1d45c728e4cffb9cb8898aede09a2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/84181974f2a147beae6c9a1f7089b681, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/53dd76b88e4349ffb85cc3bdcb88c8aa] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=37.0 K 2024-11-15T04:41:01,028 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a1d45c728e4cffb9cb8898aede09a2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731645657652 2024-11-15T04:41:01,028 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting cf12ccf9c67b4124a14bbe606bfe6bc6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731645657652 2024-11-15T04:41:01,029 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84181974f2a147beae6c9a1f7089b681, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731645658357 2024-11-15T04:41:01,029 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef38cd785e74c8cb16e827891340144, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731645658357 2024-11-15T04:41:01,029 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53dd76b88e4349ffb85cc3bdcb88c8aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731645659530 2024-11-15T04:41:01,029 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 0feaafdd2b58407b97be8facb431cde3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731645659530 2024-11-15T04:41:01,037 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:01,037 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:01,037 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/f19c549f6fa44fb48fa11c6cbd0b9add is 50, key is test_row_0/B:col10/1731645659530/Put/seqid=0 2024-11-15T04:41:01,037 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/733aaaf549a24d8a90e4bbca186f1fe6 is 50, key is test_row_0/A:col10/1731645659530/Put/seqid=0 2024-11-15T04:41:01,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742171_1347 (size=13391) 2024-11-15T04:41:01,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742172_1348 (size=13391) 2024-11-15T04:41:01,062 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/733aaaf549a24d8a90e4bbca186f1fe6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/733aaaf549a24d8a90e4bbca186f1fe6 2024-11-15T04:41:01,063 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/f19c549f6fa44fb48fa11c6cbd0b9add as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/f19c549f6fa44fb48fa11c6cbd0b9add 2024-11-15T04:41:01,068 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into 733aaaf549a24d8a90e4bbca186f1fe6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:01,068 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:41:01,068 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=13, startTime=1731645661026; duration=0sec 2024-11-15T04:41:01,068 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:01,068 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:41:01,068 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:01,070 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:01,070 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/C is initiating minor compaction (all files) 2024-11-15T04:41:01,070 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/C in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:01,070 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/7ce2b32bb2644dc18d73533f6a3597b1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/be35e01714df4c1baeecbf2410f2672e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9e81d346824847a989ac9d25bebdfc31] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=37.0 K 2024-11-15T04:41:01,070 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into f19c549f6fa44fb48fa11c6cbd0b9add(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:01,070 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:41:01,070 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ce2b32bb2644dc18d73533f6a3597b1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731645657652 2024-11-15T04:41:01,070 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=13, startTime=1731645661026; duration=0sec 2024-11-15T04:41:01,071 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:01,071 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting be35e01714df4c1baeecbf2410f2672e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731645658357 2024-11-15T04:41:01,071 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:41:01,071 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e81d346824847a989ac9d25bebdfc31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731645659530 2024-11-15T04:41:01,077 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#C#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:01,077 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/600b134e533c4d3f9db2239a88d84d5a is 50, key is test_row_0/C:col10/1731645659530/Put/seqid=0 2024-11-15T04:41:01,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742173_1349 (size=13357) 2024-11-15T04:41:01,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T04:41:01,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:01,101 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T04:41:01,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/8f73c7cc17a34c3cbf5419c5ba325511 is 50, key is test_row_0/A:col10/1731645660258/Put/seqid=0 2024-11-15T04:41:01,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742174_1350 (size=12301) 2024-11-15T04:41:01,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T04:41:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:41:01,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. as already flushing 2024-11-15T04:41:01,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645721442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:01,484 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/600b134e533c4d3f9db2239a88d84d5a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/600b134e533c4d3f9db2239a88d84d5a 2024-11-15T04:41:01,490 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/C of e5b59d814d067d3a48ee3af2799f7c44 into 600b134e533c4d3f9db2239a88d84d5a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:01,490 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:41:01,490 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/C, priority=13, startTime=1731645661026; duration=0sec 2024-11-15T04:41:01,490 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:01,490 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:41:01,509 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/8f73c7cc17a34c3cbf5419c5ba325511 2024-11-15T04:41:01,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/7a33646722b24364a801f7d467aea7c9 is 50, key is test_row_0/B:col10/1731645660258/Put/seqid=0 2024-11-15T04:41:01,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742175_1351 (size=12301) 2024-11-15T04:41:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645721553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:01,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645721762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:01,919 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/7a33646722b24364a801f7d467aea7c9 2024-11-15T04:41:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/00b8b63d00f24427bb67ea1ee6495c6a is 50, key is test_row_0/C:col10/1731645660258/Put/seqid=0 2024-11-15T04:41:01,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742176_1352 (size=12301) 2024-11-15T04:41:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54262 deadline: 1731645722072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:02,214 INFO [Thread-1174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1174. 2024-11-15T04:41:02,214 INFO [Thread-1180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1180. 2024-11-15T04:41:02,214 DEBUG [Thread-1174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:02,214 DEBUG [Thread-1180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:02,214 DEBUG [Thread-1180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,214 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,214 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:02,214 DEBUG [Thread-1180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:02,216 INFO [Thread-1178 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1178. 2024-11-15T04:41:02,216 DEBUG [Thread-1178 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:02,216 DEBUG [Thread-1178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,216 DEBUG [Thread-1178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:02,217 INFO [Thread-1176 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1176. 2024-11-15T04:41:02,217 DEBUG [Thread-1176 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:02,217 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,217 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:02,217 INFO [Thread-1182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1182. 2024-11-15T04:41:02,217 DEBUG [Thread-1182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:02,217 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,217 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:02,329 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/00b8b63d00f24427bb67ea1ee6495c6a 2024-11-15T04:41:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T04:41:02,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/8f73c7cc17a34c3cbf5419c5ba325511 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8f73c7cc17a34c3cbf5419c5ba325511 2024-11-15T04:41:02,337 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8f73c7cc17a34c3cbf5419c5ba325511, entries=150, sequenceid=456, filesize=12.0 K 2024-11-15T04:41:02,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/7a33646722b24364a801f7d467aea7c9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/7a33646722b24364a801f7d467aea7c9 2024-11-15T04:41:02,342 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/7a33646722b24364a801f7d467aea7c9, entries=150, sequenceid=456, filesize=12.0 K 2024-11-15T04:41:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/00b8b63d00f24427bb67ea1ee6495c6a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/00b8b63d00f24427bb67ea1ee6495c6a 2024-11-15T04:41:02,346 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/00b8b63d00f24427bb67ea1ee6495c6a, entries=150, sequenceid=456, filesize=12.0 K 2024-11-15T04:41:02,347 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for e5b59d814d067d3a48ee3af2799f7c44 in 1246ms, sequenceid=456, compaction requested=false 2024-11-15T04:41:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:41:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:02,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-15T04:41:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-15T04:41:02,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-15T04:41:02,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1630 sec 2024-11-15T04:41:02,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.1680 sec 2024-11-15T04:41:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:41:02,585 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:02,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T04:41:02,585 INFO [Thread-1169 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1169. 2024-11-15T04:41:02,585 DEBUG [Thread-1169 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:02,585 DEBUG [Thread-1169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,585 DEBUG [Thread-1169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:02,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:02,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/839edec40ddb45cbb86996cae0f727db is 50, key is test_row_0/A:col10/1731645662583/Put/seqid=0 2024-11-15T04:41:02,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742177_1353 (size=12301) 2024-11-15T04:41:02,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/839edec40ddb45cbb86996cae0f727db 2024-11-15T04:41:03,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/37a53205341b4062a2c520b1834dde91 is 50, key is test_row_0/B:col10/1731645662583/Put/seqid=0 2024-11-15T04:41:03,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742178_1354 (size=12301) 2024-11-15T04:41:03,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/37a53205341b4062a2c520b1834dde91 2024-11-15T04:41:03,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/deef030d75474a098b207bad0cb94b8d is 50, key is test_row_0/C:col10/1731645662583/Put/seqid=0 2024-11-15T04:41:03,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742179_1355 (size=12301) 2024-11-15T04:41:03,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/deef030d75474a098b207bad0cb94b8d 2024-11-15T04:41:03,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/839edec40ddb45cbb86996cae0f727db as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/839edec40ddb45cbb86996cae0f727db 2024-11-15T04:41:03,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/839edec40ddb45cbb86996cae0f727db, entries=150, sequenceid=472, filesize=12.0 K 2024-11-15T04:41:03,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/37a53205341b4062a2c520b1834dde91 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/37a53205341b4062a2c520b1834dde91 2024-11-15T04:41:03,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/37a53205341b4062a2c520b1834dde91, entries=150, sequenceid=472, filesize=12.0 K 2024-11-15T04:41:03,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/deef030d75474a098b207bad0cb94b8d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/deef030d75474a098b207bad0cb94b8d 2024-11-15T04:41:03,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/deef030d75474a098b207bad0cb94b8d, entries=150, sequenceid=472, filesize=12.0 K 2024-11-15T04:41:03,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for e5b59d814d067d3a48ee3af2799f7c44 in 1301ms, sequenceid=472, compaction requested=true 2024-11-15T04:41:03,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:41:03,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:03,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:03,887 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:03,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:03,887 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:03,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:03,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e5b59d814d067d3a48ee3af2799f7c44:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:03,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:03,887 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:03,887 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:03,888 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/B is initiating minor compaction (all files) 2024-11-15T04:41:03,888 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/A is initiating minor compaction (all files) 2024-11-15T04:41:03,888 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/A in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:03,888 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/B in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:03,888 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/733aaaf549a24d8a90e4bbca186f1fe6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8f73c7cc17a34c3cbf5419c5ba325511, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/839edec40ddb45cbb86996cae0f727db] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=37.1 K 2024-11-15T04:41:03,888 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/f19c549f6fa44fb48fa11c6cbd0b9add, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/7a33646722b24364a801f7d467aea7c9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/37a53205341b4062a2c520b1834dde91] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=37.1 K 2024-11-15T04:41:03,888 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 733aaaf549a24d8a90e4bbca186f1fe6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731645659530 2024-11-15T04:41:03,888 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting f19c549f6fa44fb48fa11c6cbd0b9add, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731645659530 2024-11-15T04:41:03,888 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f73c7cc17a34c3cbf5419c5ba325511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731645660205 2024-11-15T04:41:03,888 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a33646722b24364a801f7d467aea7c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731645660205 2024-11-15T04:41:03,888 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 839edec40ddb45cbb86996cae0f727db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731645661437 2024-11-15T04:41:03,888 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37a53205341b4062a2c520b1834dde91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731645661437 2024-11-15T04:41:03,894 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#A#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:03,894 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/a77ea55d430d49f3bfdcda9ff35d23b0 is 50, key is test_row_0/A:col10/1731645662583/Put/seqid=0 2024-11-15T04:41:03,899 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:03,900 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/64b2b6346c06454db46f202ebf19a225 is 50, key is test_row_0/B:col10/1731645662583/Put/seqid=0 2024-11-15T04:41:03,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742180_1356 (size=13493) 2024-11-15T04:41:03,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742181_1357 (size=13493) 2024-11-15T04:41:04,312 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/a77ea55d430d49f3bfdcda9ff35d23b0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a77ea55d430d49f3bfdcda9ff35d23b0 2024-11-15T04:41:04,312 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/64b2b6346c06454db46f202ebf19a225 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/64b2b6346c06454db46f202ebf19a225 2024-11-15T04:41:04,317 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/A of e5b59d814d067d3a48ee3af2799f7c44 into a77ea55d430d49f3bfdcda9ff35d23b0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:04,317 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/B of e5b59d814d067d3a48ee3af2799f7c44 into 64b2b6346c06454db46f202ebf19a225(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:04,317 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:41:04,317 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:41:04,317 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/B, priority=13, startTime=1731645663887; duration=0sec 2024-11-15T04:41:04,317 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/A, priority=13, startTime=1731645663886; duration=0sec 2024-11-15T04:41:04,317 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:04,317 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:B 2024-11-15T04:41:04,317 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:04,317 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:A 2024-11-15T04:41:04,317 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:04,318 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:04,318 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): e5b59d814d067d3a48ee3af2799f7c44/C is initiating minor compaction (all files) 2024-11-15T04:41:04,318 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e5b59d814d067d3a48ee3af2799f7c44/C in TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:04,318 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/600b134e533c4d3f9db2239a88d84d5a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/00b8b63d00f24427bb67ea1ee6495c6a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/deef030d75474a098b207bad0cb94b8d] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp, totalSize=37.1 K 2024-11-15T04:41:04,319 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 600b134e533c4d3f9db2239a88d84d5a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731645659530 2024-11-15T04:41:04,319 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00b8b63d00f24427bb67ea1ee6495c6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731645660205 2024-11-15T04:41:04,319 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting deef030d75474a098b207bad0cb94b8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731645661437 2024-11-15T04:41:04,324 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e5b59d814d067d3a48ee3af2799f7c44#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:04,325 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/4deb4711c01a4442aeeff5d6c647f2d8 is 50, key is test_row_0/C:col10/1731645662583/Put/seqid=0 2024-11-15T04:41:04,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742182_1358 (size=13459) 2024-11-15T04:41:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T04:41:04,343 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:04,738 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/4deb4711c01a4442aeeff5d6c647f2d8 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/4deb4711c01a4442aeeff5d6c647f2d8 2024-11-15T04:41:04,745 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e5b59d814d067d3a48ee3af2799f7c44/C of e5b59d814d067d3a48ee3af2799f7c44 into 4deb4711c01a4442aeeff5d6c647f2d8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:04,745 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e5b59d814d067d3a48ee3af2799f7c44: 2024-11-15T04:41:04,746 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., storeName=e5b59d814d067d3a48ee3af2799f7c44/C, priority=13, startTime=1731645663887; duration=0sec 2024-11-15T04:41:04,746 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:04,746 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e5b59d814d067d3a48ee3af2799f7c44:C 2024-11-15T04:41:08,526 INFO [Thread-1165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1165. 2024-11-15T04:41:08,526 DEBUG [Thread-1165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:08,526 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:08,526 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:08,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:08,547 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-11-15T04:41:08,547 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:08,547 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:08,547 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:08,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:08,566 INFO [Thread-1171 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1171. 2024-11-15T04:41:08,566 DEBUG [Thread-1171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:08,566 DEBUG [Thread-1171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:08,566 INFO [Thread-1167 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1167. 2024-11-15T04:41:08,566 DEBUG [Thread-1167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:08,567 DEBUG [Thread-1167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:08,567 DEBUG [Thread-1171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:08,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:08,567 DEBUG [Thread-1167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:08,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 199 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3628 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3738 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3626 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3632 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3730 2024-11-15T04:41:08,568 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T04:41:08,569 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T04:41:08,569 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T04:41:08,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:08,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:08,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:08,571 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T04:41:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T04:41:08,576 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645668575"}]},"ts":"1731645668575"} 2024-11-15T04:41:08,577 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T04:41:08,577 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T04:41:08,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T04:41:08,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5b59d814d067d3a48ee3af2799f7c44, UNASSIGN}] 2024-11-15T04:41:08,580 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5b59d814d067d3a48ee3af2799f7c44, UNASSIGN 2024-11-15T04:41:08,580 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=e5b59d814d067d3a48ee3af2799f7c44, regionState=CLOSING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:08,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5b59d814d067d3a48ee3af2799f7c44, UNASSIGN because future has completed 2024-11-15T04:41:08,581 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T04:41:08,581 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:41:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T04:41:08,733 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:41:08,734 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T04:41:08,734 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing e5b59d814d067d3a48ee3af2799f7c44, disabling compactions & flushes 2024-11-15T04:41:08,734 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:08,734 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:08,734 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. after waiting 0 ms 2024-11-15T04:41:08,734 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:08,734 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing e5b59d814d067d3a48ee3af2799f7c44 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-15T04:41:08,742 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/a4a614e03de4490e96b4877a21c04a03 is 50, key is test_row_1/A:col10/1731645668564/Put/seqid=0 2024-11-15T04:41:08,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742183_1359 (size=9857) 2024-11-15T04:41:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T04:41:09,148 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/a4a614e03de4490e96b4877a21c04a03 2024-11-15T04:41:09,156 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/bf52e4f4d65a4e4fbe120159e0ead4b5 is 50, key is test_row_1/B:col10/1731645668564/Put/seqid=0 2024-11-15T04:41:09,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742184_1360 (size=9857) 2024-11-15T04:41:09,160 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/bf52e4f4d65a4e4fbe120159e0ead4b5 2024-11-15T04:41:09,167 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/707b6d1624494cf5b7307356fd697f90 is 50, key is test_row_1/C:col10/1731645668564/Put/seqid=0 2024-11-15T04:41:09,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742185_1361 (size=9857) 2024-11-15T04:41:09,175 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=482 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/707b6d1624494cf5b7307356fd697f90 2024-11-15T04:41:09,180 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/A/a4a614e03de4490e96b4877a21c04a03 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a4a614e03de4490e96b4877a21c04a03 2024-11-15T04:41:09,184 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a4a614e03de4490e96b4877a21c04a03, entries=100, sequenceid=482, filesize=9.6 K 2024-11-15T04:41:09,185 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/B/bf52e4f4d65a4e4fbe120159e0ead4b5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/bf52e4f4d65a4e4fbe120159e0ead4b5 2024-11-15T04:41:09,194 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/bf52e4f4d65a4e4fbe120159e0ead4b5, entries=100, sequenceid=482, filesize=9.6 K 2024-11-15T04:41:09,195 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/.tmp/C/707b6d1624494cf5b7307356fd697f90 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/707b6d1624494cf5b7307356fd697f90 2024-11-15T04:41:09,199 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/707b6d1624494cf5b7307356fd697f90, entries=100, sequenceid=482, filesize=9.6 K 2024-11-15T04:41:09,200 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e5b59d814d067d3a48ee3af2799f7c44 in 465ms, sequenceid=482, compaction requested=false 2024-11-15T04:41:09,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b033061e768d466ba2c9e74bd347628a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d219ea45a6224438925e029f0eac5b0e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/849acc3ec8a44c158d24e52f7b86b476, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1e02103320434a92b9e253466bd44a26, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/3d1cd57166244d338d9f5c96bc668ecf, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/856141af95ff41b893ec4caec0ba4586, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1524457abdbf49928ab15481e5fe2059, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a95cddcee5254fb59dfb19907fb96584, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/88688b42c8524ca8afe8c0ce158d82eb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/4143f70760a543d49058d7e44c1a5c9c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a31a294c0c1b46768f1a08fdb8181b41, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/560c430b674544a28a10693374ce1c79, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b77ad6462ec54bd49d74ed3d5f2bd589, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8638c33422214871b17841069fd844c8, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/7c838e8731924b9ab37ccd0a80aa8117, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/678f35c6d8594f7fbf291d4f61cf51dc, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b2504518a38c4c5ba273868f3a56ad3b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/9ed46ff418f2482588b586efa66a54d5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/2f7a39ae97704dc59ceddbf6ff4c8602, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c5381a838da44fef8adb52385c46b718, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/10490c39a0e748aa8a4712fb8df4ad40, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/5eb165be0f7345f9b375d4826ba088c4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/14006e3f391741349ea12d304bc17fab, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c2a7cd18897b489788334b0967ff95b2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d1ba7ce027b44fb5923d0a722e7fcd52, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/96e77f6cc82048fe966d7c44a487c654, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d852c634ed7d44d39007a2f03d405daf, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d73a2ba625714f27963a0f9cd343d1ba, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/43f2ae9581924090a35131ec058da563, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/86a1d45c728e4cffb9cb8898aede09a2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/84181974f2a147beae6c9a1f7089b681, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/733aaaf549a24d8a90e4bbca186f1fe6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/53dd76b88e4349ffb85cc3bdcb88c8aa, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8f73c7cc17a34c3cbf5419c5ba325511, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/839edec40ddb45cbb86996cae0f727db] to archive 2024-11-15T04:41:09,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:41:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T04:41:09,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b033061e768d466ba2c9e74bd347628a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b033061e768d466ba2c9e74bd347628a 2024-11-15T04:41:09,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d219ea45a6224438925e029f0eac5b0e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d219ea45a6224438925e029f0eac5b0e 2024-11-15T04:41:09,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/849acc3ec8a44c158d24e52f7b86b476 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/849acc3ec8a44c158d24e52f7b86b476 2024-11-15T04:41:09,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1e02103320434a92b9e253466bd44a26 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1e02103320434a92b9e253466bd44a26 2024-11-15T04:41:09,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/3d1cd57166244d338d9f5c96bc668ecf to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/3d1cd57166244d338d9f5c96bc668ecf 2024-11-15T04:41:09,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/856141af95ff41b893ec4caec0ba4586 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/856141af95ff41b893ec4caec0ba4586 2024-11-15T04:41:09,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1524457abdbf49928ab15481e5fe2059 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/1524457abdbf49928ab15481e5fe2059 2024-11-15T04:41:09,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a95cddcee5254fb59dfb19907fb96584 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a95cddcee5254fb59dfb19907fb96584 2024-11-15T04:41:09,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/88688b42c8524ca8afe8c0ce158d82eb to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/88688b42c8524ca8afe8c0ce158d82eb 2024-11-15T04:41:09,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/4143f70760a543d49058d7e44c1a5c9c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/4143f70760a543d49058d7e44c1a5c9c 2024-11-15T04:41:09,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a31a294c0c1b46768f1a08fdb8181b41 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a31a294c0c1b46768f1a08fdb8181b41 2024-11-15T04:41:09,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/560c430b674544a28a10693374ce1c79 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/560c430b674544a28a10693374ce1c79 2024-11-15T04:41:09,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b77ad6462ec54bd49d74ed3d5f2bd589 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b77ad6462ec54bd49d74ed3d5f2bd589 2024-11-15T04:41:09,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8638c33422214871b17841069fd844c8 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8638c33422214871b17841069fd844c8 2024-11-15T04:41:09,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/7c838e8731924b9ab37ccd0a80aa8117 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/7c838e8731924b9ab37ccd0a80aa8117 2024-11-15T04:41:09,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/678f35c6d8594f7fbf291d4f61cf51dc to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/678f35c6d8594f7fbf291d4f61cf51dc 2024-11-15T04:41:09,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b2504518a38c4c5ba273868f3a56ad3b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/b2504518a38c4c5ba273868f3a56ad3b 2024-11-15T04:41:09,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/9ed46ff418f2482588b586efa66a54d5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/9ed46ff418f2482588b586efa66a54d5 2024-11-15T04:41:09,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/2f7a39ae97704dc59ceddbf6ff4c8602 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/2f7a39ae97704dc59ceddbf6ff4c8602 2024-11-15T04:41:09,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c5381a838da44fef8adb52385c46b718 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c5381a838da44fef8adb52385c46b718 2024-11-15T04:41:09,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/10490c39a0e748aa8a4712fb8df4ad40 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/10490c39a0e748aa8a4712fb8df4ad40 2024-11-15T04:41:09,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/5eb165be0f7345f9b375d4826ba088c4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/5eb165be0f7345f9b375d4826ba088c4 2024-11-15T04:41:09,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/14006e3f391741349ea12d304bc17fab to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/14006e3f391741349ea12d304bc17fab 2024-11-15T04:41:09,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c2a7cd18897b489788334b0967ff95b2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/c2a7cd18897b489788334b0967ff95b2 2024-11-15T04:41:09,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d1ba7ce027b44fb5923d0a722e7fcd52 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d1ba7ce027b44fb5923d0a722e7fcd52 2024-11-15T04:41:09,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/96e77f6cc82048fe966d7c44a487c654 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/96e77f6cc82048fe966d7c44a487c654 2024-11-15T04:41:09,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d852c634ed7d44d39007a2f03d405daf to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d852c634ed7d44d39007a2f03d405daf 2024-11-15T04:41:09,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d73a2ba625714f27963a0f9cd343d1ba to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/d73a2ba625714f27963a0f9cd343d1ba 2024-11-15T04:41:09,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/43f2ae9581924090a35131ec058da563 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/43f2ae9581924090a35131ec058da563 2024-11-15T04:41:09,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/86a1d45c728e4cffb9cb8898aede09a2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/86a1d45c728e4cffb9cb8898aede09a2 2024-11-15T04:41:09,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/84181974f2a147beae6c9a1f7089b681 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/84181974f2a147beae6c9a1f7089b681 2024-11-15T04:41:09,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/733aaaf549a24d8a90e4bbca186f1fe6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/733aaaf549a24d8a90e4bbca186f1fe6 2024-11-15T04:41:09,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/53dd76b88e4349ffb85cc3bdcb88c8aa to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/53dd76b88e4349ffb85cc3bdcb88c8aa 2024-11-15T04:41:09,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8f73c7cc17a34c3cbf5419c5ba325511 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/8f73c7cc17a34c3cbf5419c5ba325511 2024-11-15T04:41:09,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/839edec40ddb45cbb86996cae0f727db to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/839edec40ddb45cbb86996cae0f727db 2024-11-15T04:41:09,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/07a49cbcdc3b4c928d4bea6a321c59d4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/92e6f03b9f134d738aaac5571c02dbeb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/5568e9a37abd415db72e230668988f6a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/4f841926f0474d11ab6b55052ea0552f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/50717a862d7d41638e8effba492589df, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/a8fb07064a7d427390135dd590323ee6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b758275ce6fb4329a04ab2fc662c5b5c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9008a8df281c489fba59beee3ef87242, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b6e378322ea14a4e8e94ddf0120662d3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9d1fe5427dce4e8aa84fdf6ef15b37d0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83d635553eb84264be2ba708795eb433, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/203fc2ecbabe4907abe8b2768eb51178, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/684e89857c0548f3902f218999deb1ba, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/974b83b3fd294e8e9eb4614ec6214aa0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/6e118c35949b441aa513e47628a86d55, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/490a7577b57446f896ee0cb2cf3ca679, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/eea4df3035d846eba4e373f306098707, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/c6e0026edd26417794c72a57dd3fac39, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/1150813455194570b1ea70f1ab656c7d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/5d046a8febc34cce919f8612d90d0aac, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/fa8bdd1f88394baf9cd4aebde4de511f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/61b1ef86cad3485ebf939d7757a258f7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/78971e822078448080d151e4ac363f96, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/abc94552014a43998473bedf7f959f65, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83b092774e2546b9a109e9b9b40ae496, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/be4e3c1a23d740e7a2b8b3673521a162, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/43aadd2493bf456f92b562404870c7d5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/72e35c48941d48e490dc37bc380aacf0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/cf12ccf9c67b4124a14bbe606bfe6bc6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/ec78b22048e741afb23632c931f64b44, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/3ef38cd785e74c8cb16e827891340144, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/f19c549f6fa44fb48fa11c6cbd0b9add, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/0feaafdd2b58407b97be8facb431cde3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/7a33646722b24364a801f7d467aea7c9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/37a53205341b4062a2c520b1834dde91] to archive 2024-11-15T04:41:09,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:41:09,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/07a49cbcdc3b4c928d4bea6a321c59d4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/07a49cbcdc3b4c928d4bea6a321c59d4 2024-11-15T04:41:09,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/92e6f03b9f134d738aaac5571c02dbeb to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/92e6f03b9f134d738aaac5571c02dbeb 2024-11-15T04:41:09,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/5568e9a37abd415db72e230668988f6a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/5568e9a37abd415db72e230668988f6a 2024-11-15T04:41:09,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/4f841926f0474d11ab6b55052ea0552f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/4f841926f0474d11ab6b55052ea0552f 2024-11-15T04:41:09,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/50717a862d7d41638e8effba492589df to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/50717a862d7d41638e8effba492589df 2024-11-15T04:41:09,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/a8fb07064a7d427390135dd590323ee6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/a8fb07064a7d427390135dd590323ee6 2024-11-15T04:41:09,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b758275ce6fb4329a04ab2fc662c5b5c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b758275ce6fb4329a04ab2fc662c5b5c 2024-11-15T04:41:09,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9008a8df281c489fba59beee3ef87242 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9008a8df281c489fba59beee3ef87242 2024-11-15T04:41:09,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b6e378322ea14a4e8e94ddf0120662d3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/b6e378322ea14a4e8e94ddf0120662d3 2024-11-15T04:41:09,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9d1fe5427dce4e8aa84fdf6ef15b37d0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/9d1fe5427dce4e8aa84fdf6ef15b37d0 2024-11-15T04:41:09,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83d635553eb84264be2ba708795eb433 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83d635553eb84264be2ba708795eb433 2024-11-15T04:41:09,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/203fc2ecbabe4907abe8b2768eb51178 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/203fc2ecbabe4907abe8b2768eb51178 2024-11-15T04:41:09,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/684e89857c0548f3902f218999deb1ba to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/684e89857c0548f3902f218999deb1ba 2024-11-15T04:41:09,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/974b83b3fd294e8e9eb4614ec6214aa0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/974b83b3fd294e8e9eb4614ec6214aa0 2024-11-15T04:41:09,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/6e118c35949b441aa513e47628a86d55 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/6e118c35949b441aa513e47628a86d55 2024-11-15T04:41:09,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/490a7577b57446f896ee0cb2cf3ca679 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/490a7577b57446f896ee0cb2cf3ca679 2024-11-15T04:41:09,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/eea4df3035d846eba4e373f306098707 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/eea4df3035d846eba4e373f306098707 2024-11-15T04:41:09,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/c6e0026edd26417794c72a57dd3fac39 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/c6e0026edd26417794c72a57dd3fac39 2024-11-15T04:41:09,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/1150813455194570b1ea70f1ab656c7d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/1150813455194570b1ea70f1ab656c7d 2024-11-15T04:41:09,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/5d046a8febc34cce919f8612d90d0aac to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/5d046a8febc34cce919f8612d90d0aac 2024-11-15T04:41:09,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/fa8bdd1f88394baf9cd4aebde4de511f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/fa8bdd1f88394baf9cd4aebde4de511f 2024-11-15T04:41:09,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/61b1ef86cad3485ebf939d7757a258f7 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/61b1ef86cad3485ebf939d7757a258f7 2024-11-15T04:41:09,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/78971e822078448080d151e4ac363f96 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/78971e822078448080d151e4ac363f96 2024-11-15T04:41:09,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/abc94552014a43998473bedf7f959f65 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/abc94552014a43998473bedf7f959f65 2024-11-15T04:41:09,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83b092774e2546b9a109e9b9b40ae496 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/83b092774e2546b9a109e9b9b40ae496 2024-11-15T04:41:09,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/be4e3c1a23d740e7a2b8b3673521a162 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/be4e3c1a23d740e7a2b8b3673521a162 2024-11-15T04:41:09,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/43aadd2493bf456f92b562404870c7d5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/43aadd2493bf456f92b562404870c7d5 2024-11-15T04:41:09,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/72e35c48941d48e490dc37bc380aacf0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/72e35c48941d48e490dc37bc380aacf0 2024-11-15T04:41:09,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/cf12ccf9c67b4124a14bbe606bfe6bc6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/cf12ccf9c67b4124a14bbe606bfe6bc6 2024-11-15T04:41:09,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/ec78b22048e741afb23632c931f64b44 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/ec78b22048e741afb23632c931f64b44 2024-11-15T04:41:09,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/3ef38cd785e74c8cb16e827891340144 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/3ef38cd785e74c8cb16e827891340144 2024-11-15T04:41:09,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/f19c549f6fa44fb48fa11c6cbd0b9add to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/f19c549f6fa44fb48fa11c6cbd0b9add 2024-11-15T04:41:09,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/0feaafdd2b58407b97be8facb431cde3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/0feaafdd2b58407b97be8facb431cde3 2024-11-15T04:41:09,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/7a33646722b24364a801f7d467aea7c9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/7a33646722b24364a801f7d467aea7c9 2024-11-15T04:41:09,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/37a53205341b4062a2c520b1834dde91 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/37a53205341b4062a2c520b1834dde91 2024-11-15T04:41:09,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e762ecac47524d14a21052328ede9c0b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/75292478ddb145dbb7d3b1b73f6b7bb9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/efe44a39d9004b9984b2c8c7a3b713b0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/f172e84322cf44169a89403ae8edeae7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/edcd9b95c68747b9999f7e2d9f9ace9a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cbab6f58be554914a4a1b7394d802d02, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2b10d68dad0f4785ab276f34bf752196, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/5d45e3bfac044a59a26e34f13760aec9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cfa985dc0e3c4df7be4e24f36bd1332c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/b3ca0075e8ea4d38b828971b10a47e49, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2725d18573264048af2048c385aa6481, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/fa27a161676640e5bb66b1d115808be4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d143656edd0d483ca63489eaf110a77b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2687a00026644a74907b8e7ded765cf6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2f77a3908880414eaee7a9280e580b4f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e81ae65c355549fe93b7c2f3dc92be19, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/b51d12bfe8834520b5488e94e5ef4176, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/313676e69b8c4da9a451da24a86ffee5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d170e9b9fa904a56b11f89e0865cea01, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d9d99a3b3f6747d69f1bef87056ed9c4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/27c48efcc72943599456216da03fdfde, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ab658f0c454b4e45a244cd4168a7791a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9ac6509fe1784010bee3ed54aa09b466, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9a71a6c19814c1dbb66084ee6e20e8c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9d03e2a29e194d47b7ea21683a707dbb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ec0b7980813041109c9f3c8ccaa731fe, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d938455e058346e5af3c899609c27d0a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/7ce2b32bb2644dc18d73533f6a3597b1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9b08f3631424254a6d10ccaef583922, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/be35e01714df4c1baeecbf2410f2672e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/600b134e533c4d3f9db2239a88d84d5a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9e81d346824847a989ac9d25bebdfc31, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/00b8b63d00f24427bb67ea1ee6495c6a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/deef030d75474a098b207bad0cb94b8d] to archive 2024-11-15T04:41:09,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:41:09,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e762ecac47524d14a21052328ede9c0b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e762ecac47524d14a21052328ede9c0b 2024-11-15T04:41:09,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/75292478ddb145dbb7d3b1b73f6b7bb9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/75292478ddb145dbb7d3b1b73f6b7bb9 2024-11-15T04:41:09,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/efe44a39d9004b9984b2c8c7a3b713b0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/efe44a39d9004b9984b2c8c7a3b713b0 2024-11-15T04:41:09,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/f172e84322cf44169a89403ae8edeae7 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/f172e84322cf44169a89403ae8edeae7 2024-11-15T04:41:09,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/edcd9b95c68747b9999f7e2d9f9ace9a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/edcd9b95c68747b9999f7e2d9f9ace9a 2024-11-15T04:41:09,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cbab6f58be554914a4a1b7394d802d02 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cbab6f58be554914a4a1b7394d802d02 2024-11-15T04:41:09,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2b10d68dad0f4785ab276f34bf752196 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2b10d68dad0f4785ab276f34bf752196 2024-11-15T04:41:09,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/5d45e3bfac044a59a26e34f13760aec9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/5d45e3bfac044a59a26e34f13760aec9 2024-11-15T04:41:09,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cfa985dc0e3c4df7be4e24f36bd1332c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/cfa985dc0e3c4df7be4e24f36bd1332c 2024-11-15T04:41:09,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/b3ca0075e8ea4d38b828971b10a47e49 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/b3ca0075e8ea4d38b828971b10a47e49 2024-11-15T04:41:09,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2725d18573264048af2048c385aa6481 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2725d18573264048af2048c385aa6481 2024-11-15T04:41:09,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/fa27a161676640e5bb66b1d115808be4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/fa27a161676640e5bb66b1d115808be4 2024-11-15T04:41:09,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d143656edd0d483ca63489eaf110a77b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d143656edd0d483ca63489eaf110a77b 2024-11-15T04:41:09,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2687a00026644a74907b8e7ded765cf6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2687a00026644a74907b8e7ded765cf6 2024-11-15T04:41:09,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2f77a3908880414eaee7a9280e580b4f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/2f77a3908880414eaee7a9280e580b4f 2024-11-15T04:41:09,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e81ae65c355549fe93b7c2f3dc92be19 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/e81ae65c355549fe93b7c2f3dc92be19 2024-11-15T04:41:09,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/b51d12bfe8834520b5488e94e5ef4176 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/b51d12bfe8834520b5488e94e5ef4176 2024-11-15T04:41:09,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/313676e69b8c4da9a451da24a86ffee5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/313676e69b8c4da9a451da24a86ffee5 2024-11-15T04:41:09,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d170e9b9fa904a56b11f89e0865cea01 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d170e9b9fa904a56b11f89e0865cea01 2024-11-15T04:41:09,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d9d99a3b3f6747d69f1bef87056ed9c4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d9d99a3b3f6747d69f1bef87056ed9c4 2024-11-15T04:41:09,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/27c48efcc72943599456216da03fdfde to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/27c48efcc72943599456216da03fdfde 2024-11-15T04:41:09,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ab658f0c454b4e45a244cd4168a7791a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ab658f0c454b4e45a244cd4168a7791a 2024-11-15T04:41:09,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9ac6509fe1784010bee3ed54aa09b466 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9ac6509fe1784010bee3ed54aa09b466 2024-11-15T04:41:09,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9a71a6c19814c1dbb66084ee6e20e8c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9a71a6c19814c1dbb66084ee6e20e8c 2024-11-15T04:41:09,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9d03e2a29e194d47b7ea21683a707dbb to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9d03e2a29e194d47b7ea21683a707dbb 2024-11-15T04:41:09,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ec0b7980813041109c9f3c8ccaa731fe to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/ec0b7980813041109c9f3c8ccaa731fe 2024-11-15T04:41:09,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d938455e058346e5af3c899609c27d0a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/d938455e058346e5af3c899609c27d0a 2024-11-15T04:41:09,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/7ce2b32bb2644dc18d73533f6a3597b1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/7ce2b32bb2644dc18d73533f6a3597b1 2024-11-15T04:41:09,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9b08f3631424254a6d10ccaef583922 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/a9b08f3631424254a6d10ccaef583922 2024-11-15T04:41:09,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/be35e01714df4c1baeecbf2410f2672e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/be35e01714df4c1baeecbf2410f2672e 2024-11-15T04:41:09,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/600b134e533c4d3f9db2239a88d84d5a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/600b134e533c4d3f9db2239a88d84d5a 2024-11-15T04:41:09,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9e81d346824847a989ac9d25bebdfc31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/9e81d346824847a989ac9d25bebdfc31 2024-11-15T04:41:09,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/00b8b63d00f24427bb67ea1ee6495c6a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/00b8b63d00f24427bb67ea1ee6495c6a 2024-11-15T04:41:09,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/deef030d75474a098b207bad0cb94b8d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/deef030d75474a098b207bad0cb94b8d 2024-11-15T04:41:09,340 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/recovered.edits/485.seqid, newMaxSeqId=485, maxSeqId=1 2024-11-15T04:41:09,340 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. 2024-11-15T04:41:09,340 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for e5b59d814d067d3a48ee3af2799f7c44: Waiting for close lock at 1731645668734Running coprocessor pre-close hooks at 1731645668734Disabling compacts and flushes for region at 1731645668734Disabling writes for close at 1731645668734Obtaining lock to block concurrent updates at 1731645668734Preparing flush snapshotting stores in e5b59d814d067d3a48ee3af2799f7c44 at 1731645668734Finished memstore snapshotting TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731645668735 (+1 ms)Flushing stores of TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44. at 1731645668737 (+2 ms)Flushing e5b59d814d067d3a48ee3af2799f7c44/A: creating writer at 1731645668737Flushing e5b59d814d067d3a48ee3af2799f7c44/A: appending metadata at 1731645668742 (+5 ms)Flushing e5b59d814d067d3a48ee3af2799f7c44/A: closing flushed file at 1731645668742Flushing e5b59d814d067d3a48ee3af2799f7c44/B: creating writer at 1731645669153 (+411 ms)Flushing e5b59d814d067d3a48ee3af2799f7c44/B: appending metadata at 1731645669155 (+2 ms)Flushing e5b59d814d067d3a48ee3af2799f7c44/B: closing flushed file at 1731645669156 (+1 ms)Flushing e5b59d814d067d3a48ee3af2799f7c44/C: creating writer at 1731645669164 (+8 ms)Flushing e5b59d814d067d3a48ee3af2799f7c44/C: appending metadata at 1731645669167 (+3 ms)Flushing e5b59d814d067d3a48ee3af2799f7c44/C: closing flushed file at 1731645669167Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d30c5ae: reopening flushed file at 1731645669179 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bfd2c52: reopening flushed file at 1731645669184 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5811c222: reopening flushed file at 1731645669194 (+10 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e5b59d814d067d3a48ee3af2799f7c44 in 465ms, sequenceid=482, compaction requested=false at 1731645669200 (+6 ms)Writing region close event to WAL at 1731645669336 (+136 ms)Running coprocessor post-close hooks at 1731645669340 (+4 ms)Closed at 1731645669340 2024-11-15T04:41:09,342 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:41:09,342 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=e5b59d814d067d3a48ee3af2799f7c44, regionState=CLOSED 2024-11-15T04:41:09,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:41:09,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-15T04:41:09,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure e5b59d814d067d3a48ee3af2799f7c44, server=26747e6ada4f,42935,1731645581086 in 763 msec 2024-11-15T04:41:09,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-15T04:41:09,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e5b59d814d067d3a48ee3af2799f7c44, UNASSIGN in 766 msec 2024-11-15T04:41:09,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-15T04:41:09,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 769 msec 2024-11-15T04:41:09,349 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645669349"}]},"ts":"1731645669349"} 2024-11-15T04:41:09,351 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T04:41:09,351 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T04:41:09,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 780 msec 2024-11-15T04:41:09,361 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T04:41:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T04:41:09,713 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:09,714 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T04:41:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:09,715 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T04:41:09,716 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:09,718 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:41:09,720 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/recovered.edits] 2024-11-15T04:41:09,722 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a4a614e03de4490e96b4877a21c04a03 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a4a614e03de4490e96b4877a21c04a03 2024-11-15T04:41:09,723 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a77ea55d430d49f3bfdcda9ff35d23b0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/A/a77ea55d430d49f3bfdcda9ff35d23b0 2024-11-15T04:41:09,725 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/64b2b6346c06454db46f202ebf19a225 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/64b2b6346c06454db46f202ebf19a225 2024-11-15T04:41:09,726 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/bf52e4f4d65a4e4fbe120159e0ead4b5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/B/bf52e4f4d65a4e4fbe120159e0ead4b5 2024-11-15T04:41:09,728 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/4deb4711c01a4442aeeff5d6c647f2d8 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/4deb4711c01a4442aeeff5d6c647f2d8 2024-11-15T04:41:09,729 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/707b6d1624494cf5b7307356fd697f90 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/C/707b6d1624494cf5b7307356fd697f90 2024-11-15T04:41:09,732 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/recovered.edits/485.seqid to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44/recovered.edits/485.seqid 2024-11-15T04:41:09,732 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/e5b59d814d067d3a48ee3af2799f7c44 2024-11-15T04:41:09,732 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T04:41:09,734 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:09,738 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T04:41:09,741 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T04:41:09,742 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:09,742 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T04:41:09,742 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731645669742"}]},"ts":"9223372036854775807"} 2024-11-15T04:41:09,744 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T04:41:09,744 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e5b59d814d067d3a48ee3af2799f7c44, NAME => 'TestAcidGuarantees,,1731645640008.e5b59d814d067d3a48ee3af2799f7c44.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T04:41:09,744 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T04:41:09,745 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731645669745"}]},"ts":"9223372036854775807"} 2024-11-15T04:41:09,747 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T04:41:09,748 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:09,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-15T04:41:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T04:41:09,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T04:41:09,825 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:09,833 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 242), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=535 (was 529) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=14198 (was 14681) 2024-11-15T04:41:09,842 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=535, ProcessCount=11, AvailableMemoryMB=14198 2024-11-15T04:41:09,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T04:41:09,844 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T04:41:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:09,846 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T04:41:09,846 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:09,846 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-15T04:41:09,847 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T04:41:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T04:41:09,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742186_1362 (size=959) 2024-11-15T04:41:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T04:41:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T04:41:10,253 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6 2024-11-15T04:41:10,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742187_1363 (size=53) 2024-11-15T04:41:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T04:41:10,637 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T04:41:10,660 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:41:10,661 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 05250181cbe972c6b3047f25758a3996, disabling compactions & flushes 2024-11-15T04:41:10,661 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:10,661 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:10,661 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. after waiting 0 ms 2024-11-15T04:41:10,661 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:10,661 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:10,661 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 05250181cbe972c6b3047f25758a3996: Waiting for close lock at 1731645670660Disabling compacts and flushes for region at 1731645670660Disabling writes for close at 1731645670661 (+1 ms)Writing region close event to WAL at 1731645670661Closed at 1731645670661 2024-11-15T04:41:10,663 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T04:41:10,664 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731645670663"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731645670663"}]},"ts":"1731645670663"} 2024-11-15T04:41:10,666 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T04:41:10,668 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T04:41:10,668 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645670668"}]},"ts":"1731645670668"} 2024-11-15T04:41:10,670 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T04:41:10,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, ASSIGN}] 2024-11-15T04:41:10,672 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, ASSIGN 2024-11-15T04:41:10,673 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, ASSIGN; state=OFFLINE, location=26747e6ada4f,42935,1731645581086; forceNewPlan=false, retain=false 2024-11-15T04:41:10,824 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=05250181cbe972c6b3047f25758a3996, regionState=OPENING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:10,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, ASSIGN because future has completed 2024-11-15T04:41:10,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:41:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T04:41:10,991 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:10,991 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:41:10,992 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:10,992 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:41:10,992 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:10,992 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:10,995 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:10,997 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05250181cbe972c6b3047f25758a3996 columnFamilyName A 2024-11-15T04:41:10,998 DEBUG [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:10,998 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(327): Store=05250181cbe972c6b3047f25758a3996/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:41:10,999 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:11,000 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05250181cbe972c6b3047f25758a3996 columnFamilyName B 2024-11-15T04:41:11,000 DEBUG [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:11,001 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(327): Store=05250181cbe972c6b3047f25758a3996/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:41:11,001 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:11,002 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05250181cbe972c6b3047f25758a3996 columnFamilyName C 2024-11-15T04:41:11,002 DEBUG [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:11,003 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(327): Store=05250181cbe972c6b3047f25758a3996/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:41:11,003 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:11,004 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:11,005 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:11,006 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:11,006 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:11,007 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:41:11,008 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:11,010 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T04:41:11,011 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 05250181cbe972c6b3047f25758a3996; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63154717, jitterRate=-0.05892138183116913}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:41:11,011 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:11,011 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 05250181cbe972c6b3047f25758a3996: Running coprocessor pre-open hook at 1731645670992Writing region info on filesystem at 1731645670992Initializing all the Stores at 1731645670994 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645670994Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645670994Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645670995 (+1 ms)Cleaning up temporary data from old regions at 1731645671006 (+11 ms)Running coprocessor post-open hooks at 1731645671011 (+5 ms)Region opened successfully at 1731645671011 2024-11-15T04:41:11,012 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., pid=89, masterSystemTime=1731645670984 2024-11-15T04:41:11,014 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:11,014 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:11,014 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=05250181cbe972c6b3047f25758a3996, regionState=OPEN, openSeqNum=2, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:11,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:41:11,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-15T04:41:11,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 in 187 msec 2024-11-15T04:41:11,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-15T04:41:11,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, ASSIGN in 347 msec 2024-11-15T04:41:11,020 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T04:41:11,020 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645671020"}]},"ts":"1731645671020"} 2024-11-15T04:41:11,021 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T04:41:11,022 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T04:41:11,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-15T04:41:11,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T04:41:11,995 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:11,997 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680843a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:11,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:11,997 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:11,999 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a33704d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,003 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239988c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,004 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,008 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T04:41:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 26747e6ada4f,42655,1731645580243 2024-11-15T04:41:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c36b152 2024-11-15T04:41:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T04:41:12,010 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T04:41:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T04:41:12,013 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T04:41:12,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:12,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742188_1364 (size=995) 2024-11-15T04:41:12,433 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T04:41:12,433 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T04:41:12,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T04:41:12,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, REOPEN/MOVE}] 2024-11-15T04:41:12,446 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, REOPEN/MOVE 2024-11-15T04:41:12,447 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=05250181cbe972c6b3047f25758a3996, regionState=CLOSING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:12,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, REOPEN/MOVE because future has completed 2024-11-15T04:41:12,449 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T04:41:12,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:41:12,601 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,601 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T04:41:12,601 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 05250181cbe972c6b3047f25758a3996, disabling compactions & flushes 2024-11-15T04:41:12,601 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:12,601 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:12,601 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. after waiting 0 ms 2024-11-15T04:41:12,601 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:12,604 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T04:41:12,605 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:12,605 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 05250181cbe972c6b3047f25758a3996: Waiting for close lock at 1731645672601Running coprocessor pre-close hooks at 1731645672601Disabling compacts and flushes for region at 1731645672601Disabling writes for close at 1731645672601Writing region close event to WAL at 1731645672601Running coprocessor post-close hooks at 1731645672605 (+4 ms)Closed at 1731645672605 2024-11-15T04:41:12,605 WARN [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 05250181cbe972c6b3047f25758a3996 to self. 2024-11-15T04:41:12,607 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,608 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=05250181cbe972c6b3047f25758a3996, regionState=CLOSED 2024-11-15T04:41:12,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:41:12,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-15T04:41:12,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 in 161 msec 2024-11-15T04:41:12,613 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, REOPEN/MOVE; state=CLOSED, location=26747e6ada4f,42935,1731645581086; forceNewPlan=false, retain=true 2024-11-15T04:41:12,764 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=05250181cbe972c6b3047f25758a3996, regionState=OPENING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:12,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, REOPEN/MOVE because future has completed 2024-11-15T04:41:12,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:41:12,920 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:12,920 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:41:12,921 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,921 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:41:12,921 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,921 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,922 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,923 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05250181cbe972c6b3047f25758a3996 columnFamilyName A 2024-11-15T04:41:12,923 DEBUG [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:12,924 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(327): Store=05250181cbe972c6b3047f25758a3996/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:41:12,924 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,925 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05250181cbe972c6b3047f25758a3996 columnFamilyName B 2024-11-15T04:41:12,925 DEBUG [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:12,925 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(327): Store=05250181cbe972c6b3047f25758a3996/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:41:12,925 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,926 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05250181cbe972c6b3047f25758a3996 columnFamilyName C 2024-11-15T04:41:12,926 DEBUG [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:12,926 INFO [StoreOpener-05250181cbe972c6b3047f25758a3996-1 {}] regionserver.HStore(327): Store=05250181cbe972c6b3047f25758a3996/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:41:12,926 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,927 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,928 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,928 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,928 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,929 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:41:12,930 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,930 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 05250181cbe972c6b3047f25758a3996; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75081790, jitterRate=0.11880585551261902}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:41:12,930 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,931 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 05250181cbe972c6b3047f25758a3996: Running coprocessor pre-open hook at 1731645672921Writing region info on filesystem at 1731645672921Initializing all the Stores at 1731645672922 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645672922Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645672922Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645672922Cleaning up temporary data from old regions at 1731645672928 (+6 ms)Running coprocessor post-open hooks at 1731645672930 (+2 ms)Region opened successfully at 1731645672931 (+1 ms) 2024-11-15T04:41:12,931 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., pid=94, masterSystemTime=1731645672917 2024-11-15T04:41:12,933 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:12,933 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:12,934 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=05250181cbe972c6b3047f25758a3996, regionState=OPEN, openSeqNum=5, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:12,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:41:12,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-15T04:41:12,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 in 169 msec 2024-11-15T04:41:12,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-15T04:41:12,939 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, REOPEN/MOVE in 492 msec 2024-11-15T04:41:12,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-15T04:41:12,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-11-15T04:41:12,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 928 msec 2024-11-15T04:41:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T04:41:12,943 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:12,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f539a03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:12,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:12,944 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366144a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,946 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,946 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba4cd98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43ef673d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:12,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:12,947 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a2a057f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,949 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7978e9ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc7c0e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,950 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:12,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:12,951 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97f6c9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,952 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20574e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5958f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:12,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:12,954 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25a22845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,955 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4410f8fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41325ea4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:12,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:12,957 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35974178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,958 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c550df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a06cc9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:12,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:12,960 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e39ce2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,961 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7888b3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bdde2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:12,962 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:12,963 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c26906, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,964 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da2eaa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7608485b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:12,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:12,966 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ecb1c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,967 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bf954ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf0e288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:12,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:12,969 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4ce128, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,969 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,970 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567601f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3f81f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:12,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:12,972 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@220ed3ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:12,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:12,973 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:12,973 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1371090f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:12,974 DEBUG [Thread-1651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,974 DEBUG [Thread-1649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,974 DEBUG [Thread-1656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,974 DEBUG [Thread-1658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,975 DEBUG [Thread-1654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,975 DEBUG [Thread-1660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,975 DEBUG [Thread-1645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,977 DEBUG [Thread-1643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,977 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:41:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:41:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:41:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:41:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:41:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:41:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:41:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:41:12,987 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:12,987 DEBUG [Thread-1647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-15T04:41:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,989 DEBUG [Thread-1662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T04:41:12,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:12,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:12,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,991 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:12,991 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:12,993 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:12,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:41:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:41:12,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:41:12,995 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:12,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:13,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155b152c1b04aa4d8ca20745e76537069c_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645672981/Put/seqid=0 2024-11-15T04:41:13,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645733031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645733032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645733032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645733037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742189_1365 (size=14594) 2024-11-15T04:41:13,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:13,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645733041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,050 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155b152c1b04aa4d8ca20745e76537069c_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155b152c1b04aa4d8ca20745e76537069c_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:13,051 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/9af6cca0a9f8432f84c208f81cbb14bd, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:13,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/9af6cca0a9f8432f84c208f81cbb14bd is 175, key is test_row_0/A:col10/1731645672981/Put/seqid=0 2024-11-15T04:41:13,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742190_1366 (size=39549) 2024-11-15T04:41:13,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/9af6cca0a9f8432f84c208f81cbb14bd 2024-11-15T04:41:13,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/2445c0237f0b40ebb7b321de0b40975c is 50, key is test_row_0/B:col10/1731645672981/Put/seqid=0 2024-11-15T04:41:13,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742191_1367 (size=12001) 2024-11-15T04:41:13,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T04:41:13,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T04:41:13,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645733152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645733153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645733153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645733153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645733153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T04:41:13,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:13,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T04:41:13,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645733363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645733363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645733363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645733363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645733363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T04:41:13,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:13,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/2445c0237f0b40ebb7b321de0b40975c 2024-11-15T04:41:13,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/d2febb30c8c842e28025f1cc9a36a8d1 is 50, key is test_row_0/C:col10/1731645672981/Put/seqid=0 2024-11-15T04:41:13,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742192_1368 (size=12001) 2024-11-15T04:41:13,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T04:41:13,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:13,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T04:41:13,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645733672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645733672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645733673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645733673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645733683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:13,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:13,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T04:41:13,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:13,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/d2febb30c8c842e28025f1cc9a36a8d1 2024-11-15T04:41:13,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/9af6cca0a9f8432f84c208f81cbb14bd as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/9af6cca0a9f8432f84c208f81cbb14bd 2024-11-15T04:41:13,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T04:41:13,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:13,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:13,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:13,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/9af6cca0a9f8432f84c208f81cbb14bd, entries=200, sequenceid=15, filesize=38.6 K 2024-11-15T04:41:13,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/2445c0237f0b40ebb7b321de0b40975c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/2445c0237f0b40ebb7b321de0b40975c 2024-11-15T04:41:13,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/2445c0237f0b40ebb7b321de0b40975c, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T04:41:13,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/d2febb30c8c842e28025f1cc9a36a8d1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/d2febb30c8c842e28025f1cc9a36a8d1 2024-11-15T04:41:13,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/d2febb30c8c842e28025f1cc9a36a8d1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T04:41:13,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 05250181cbe972c6b3047f25758a3996 in 929ms, sequenceid=15, compaction requested=false 2024-11-15T04:41:13,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:14,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T04:41:14,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:14,065 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:41:14,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111563b16297dc2845afa1a468550e608fec_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645673035/Put/seqid=0 2024-11-15T04:41:14,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742193_1369 (size=12154) 2024-11-15T04:41:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T04:41:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:14,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:14,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645734193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645734193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645734194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645734196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645734199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645734303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645734312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645734312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645734312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:14,486 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111563b16297dc2845afa1a468550e608fec_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111563b16297dc2845afa1a468550e608fec_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:14,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/6e098871a1c64664b2e0e2128ed4274d, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:14,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/6e098871a1c64664b2e0e2128ed4274d is 175, key is test_row_0/A:col10/1731645673035/Put/seqid=0 2024-11-15T04:41:14,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742194_1370 (size=30955) 2024-11-15T04:41:14,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645734512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645734533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645734533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645734533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645734822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645734853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645734853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645734854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:14,873 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T04:41:14,897 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/6e098871a1c64664b2e0e2128ed4274d 2024-11-15T04:41:14,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/fc78ed97ddd642c2b67b7aa8fdce7e72 is 50, key is test_row_0/B:col10/1731645673035/Put/seqid=0 2024-11-15T04:41:14,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742195_1371 (size=12001) 2024-11-15T04:41:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T04:41:15,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645735213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:15,224 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in 05250181cbe972c6b3047f25758a3996 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:41:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:15,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:15,338 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/fc78ed97ddd642c2b67b7aa8fdce7e72 2024-11-15T04:41:15,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/c65d99bbf3274a9e922bc94cfb7d6083 is 50, key is test_row_0/C:col10/1731645673035/Put/seqid=0 2024-11-15T04:41:15,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645735343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:15,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742196_1372 (size=12001) 2024-11-15T04:41:15,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:15,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:15,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645735373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:15,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645735373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:15,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645735373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:15,752 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/c65d99bbf3274a9e922bc94cfb7d6083 2024-11-15T04:41:15,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/6e098871a1c64664b2e0e2128ed4274d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/6e098871a1c64664b2e0e2128ed4274d 2024-11-15T04:41:15,759 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/6e098871a1c64664b2e0e2128ed4274d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-15T04:41:15,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/fc78ed97ddd642c2b67b7aa8fdce7e72 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/fc78ed97ddd642c2b67b7aa8fdce7e72 2024-11-15T04:41:15,763 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/fc78ed97ddd642c2b67b7aa8fdce7e72, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T04:41:15,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/c65d99bbf3274a9e922bc94cfb7d6083 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/c65d99bbf3274a9e922bc94cfb7d6083 2024-11-15T04:41:15,767 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/c65d99bbf3274a9e922bc94cfb7d6083, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T04:41:15,768 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 05250181cbe972c6b3047f25758a3996 in 1703ms, sequenceid=40, compaction requested=false 2024-11-15T04:41:15,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:15,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:15,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-15T04:41:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-15T04:41:15,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-15T04:41:15,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7740 sec 2024-11-15T04:41:15,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.7850 sec 2024-11-15T04:41:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:16,369 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:16,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T04:41:16,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111536ffd46d74ca4eb99a6b60a9646ca0f6_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645674193/Put/seqid=0 2024-11-15T04:41:16,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742197_1373 (size=14594) 2024-11-15T04:41:16,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645736454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645736454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645736465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645736466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645736572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645736572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645736582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645736582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,780 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:16,784 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111536ffd46d74ca4eb99a6b60a9646ca0f6_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111536ffd46d74ca4eb99a6b60a9646ca0f6_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:16,784 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/8cfbb1b1877744788be2a4f040321ffd, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:16,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/8cfbb1b1877744788be2a4f040321ffd is 175, key is test_row_0/A:col10/1731645674193/Put/seqid=0 2024-11-15T04:41:16,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742198_1374 (size=39549) 2024-11-15T04:41:16,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645736783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645736792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645736792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645736792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:17,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645737103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:17,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:17,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645737103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645737103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:17,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:17,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645737103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T04:41:17,143 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:17,143 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-15T04:41:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T04:41:17,145 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:17,145 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:17,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:17,189 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/8cfbb1b1877744788be2a4f040321ffd 2024-11-15T04:41:17,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/f821fe44fe334e6c8a84fd929cf2171e is 50, key is test_row_0/B:col10/1731645674193/Put/seqid=0 2024-11-15T04:41:17,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742199_1375 (size=12001) 2024-11-15T04:41:17,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:17,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645737242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:17,246 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in 05250181cbe972c6b3047f25758a3996 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:41:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T04:41:17,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T04:41:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:17,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T04:41:17,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:17,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T04:41:17,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/f821fe44fe334e6c8a84fd929cf2171e 2024-11-15T04:41:17,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T04:41:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:17,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/2373960e75dc4143841d101fd4acce84 is 50, key is test_row_0/C:col10/1731645674193/Put/seqid=0 2024-11-15T04:41:17,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742200_1376 (size=12001) 2024-11-15T04:41:17,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645737612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645737612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:17,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645737612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645737613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:17,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T04:41:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:17,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T04:41:17,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T04:41:17,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:17,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:17,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:17,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:17,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:18,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/2373960e75dc4143841d101fd4acce84 2024-11-15T04:41:18,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/8cfbb1b1877744788be2a4f040321ffd as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8cfbb1b1877744788be2a4f040321ffd 2024-11-15T04:41:18,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8cfbb1b1877744788be2a4f040321ffd, entries=200, sequenceid=52, filesize=38.6 K 2024-11-15T04:41:18,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/f821fe44fe334e6c8a84fd929cf2171e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f821fe44fe334e6c8a84fd929cf2171e 2024-11-15T04:41:18,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f821fe44fe334e6c8a84fd929cf2171e, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T04:41:18,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/2373960e75dc4143841d101fd4acce84 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/2373960e75dc4143841d101fd4acce84 2024-11-15T04:41:18,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/2373960e75dc4143841d101fd4acce84, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T04:41:18,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 05250181cbe972c6b3047f25758a3996 in 1658ms, sequenceid=52, compaction requested=true 2024-11-15T04:41:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:18,027 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:18,027 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:18,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:18,028 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:18,028 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:18,028 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/A is initiating minor compaction (all files) 2024-11-15T04:41:18,028 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/B is initiating minor compaction (all files) 2024-11-15T04:41:18,028 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/A in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:18,028 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/B in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:18,028 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/9af6cca0a9f8432f84c208f81cbb14bd, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/6e098871a1c64664b2e0e2128ed4274d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8cfbb1b1877744788be2a4f040321ffd] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=107.5 K 2024-11-15T04:41:18,028 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/2445c0237f0b40ebb7b321de0b40975c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/fc78ed97ddd642c2b67b7aa8fdce7e72, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f821fe44fe334e6c8a84fd929cf2171e] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=35.2 K 2024-11-15T04:41:18,028 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:18,028 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/9af6cca0a9f8432f84c208f81cbb14bd, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/6e098871a1c64664b2e0e2128ed4274d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8cfbb1b1877744788be2a4f040321ffd] 2024-11-15T04:41:18,028 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 2445c0237f0b40ebb7b321de0b40975c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731645672981 2024-11-15T04:41:18,029 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9af6cca0a9f8432f84c208f81cbb14bd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731645672981 2024-11-15T04:41:18,029 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting fc78ed97ddd642c2b67b7aa8fdce7e72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731645673030 2024-11-15T04:41:18,029 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e098871a1c64664b2e0e2128ed4274d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731645673030 2024-11-15T04:41:18,029 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting f821fe44fe334e6c8a84fd929cf2171e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645674189 2024-11-15T04:41:18,029 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cfbb1b1877744788be2a4f040321ffd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645674189 2024-11-15T04:41:18,037 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:18,038 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:18,039 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/4d218eb505334a03811b313cd379d359 is 50, key is test_row_0/B:col10/1731645674193/Put/seqid=0 2024-11-15T04:41:18,039 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115872b8861ec604dc1a01beec6db440d1e_05250181cbe972c6b3047f25758a3996 store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:18,041 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115872b8861ec604dc1a01beec6db440d1e_05250181cbe972c6b3047f25758a3996, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:18,041 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115872b8861ec604dc1a01beec6db440d1e_05250181cbe972c6b3047f25758a3996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:18,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742201_1377 (size=12104) 2024-11-15T04:41:18,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742202_1378 (size=4469) 2024-11-15T04:41:18,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T04:41:18,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:18,063 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T04:41:18,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b4755699c5ee47469c7dd2e8272dc692_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645676453/Put/seqid=0 2024-11-15T04:41:18,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742203_1379 (size=12154) 2024-11-15T04:41:18,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T04:41:18,445 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#A#compaction#321 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:18,446 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/60f05b03dfeb4f58999ddf3e4272e23c is 175, key is test_row_0/A:col10/1731645674193/Put/seqid=0 2024-11-15T04:41:18,448 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/4d218eb505334a03811b313cd379d359 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/4d218eb505334a03811b313cd379d359 2024-11-15T04:41:18,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742204_1380 (size=31058) 2024-11-15T04:41:18,455 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/B of 05250181cbe972c6b3047f25758a3996 into 4d218eb505334a03811b313cd379d359(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:18,455 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:18,455 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/B, priority=13, startTime=1731645678027; duration=0sec 2024-11-15T04:41:18,455 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:18,455 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:B 2024-11-15T04:41:18,455 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:18,456 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:18,456 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/C is initiating minor compaction (all files) 2024-11-15T04:41:18,456 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/C in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:18,456 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/d2febb30c8c842e28025f1cc9a36a8d1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/c65d99bbf3274a9e922bc94cfb7d6083, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/2373960e75dc4143841d101fd4acce84] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=35.2 K 2024-11-15T04:41:18,457 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting d2febb30c8c842e28025f1cc9a36a8d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731645672981 2024-11-15T04:41:18,457 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c65d99bbf3274a9e922bc94cfb7d6083, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731645673030 2024-11-15T04:41:18,457 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 2373960e75dc4143841d101fd4acce84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645674189 2024-11-15T04:41:18,463 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:18,463 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/1eb1666b7c4d4b5fba5872d530baac3d is 50, key is test_row_0/C:col10/1731645674193/Put/seqid=0 2024-11-15T04:41:18,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742205_1381 (size=12104) 2024-11-15T04:41:18,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:18,478 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b4755699c5ee47469c7dd2e8272dc692_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4755699c5ee47469c7dd2e8272dc692_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:18,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/d834aad3ccce48a8ae69408154411663, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:18,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/d834aad3ccce48a8ae69408154411663 is 175, key is test_row_0/A:col10/1731645676453/Put/seqid=0 2024-11-15T04:41:18,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742206_1382 (size=30955) 2024-11-15T04:41:18,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:18,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:18,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645738639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645738642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645738649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645738650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645738753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645738753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645738763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645738763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,854 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/60f05b03dfeb4f58999ddf3e4272e23c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/60f05b03dfeb4f58999ddf3e4272e23c 2024-11-15T04:41:18,860 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/A of 05250181cbe972c6b3047f25758a3996 into 60f05b03dfeb4f58999ddf3e4272e23c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:18,860 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:18,860 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/A, priority=13, startTime=1731645678027; duration=0sec 2024-11-15T04:41:18,860 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:18,861 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:A 2024-11-15T04:41:18,872 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/1eb1666b7c4d4b5fba5872d530baac3d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/1eb1666b7c4d4b5fba5872d530baac3d 2024-11-15T04:41:18,876 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/C of 05250181cbe972c6b3047f25758a3996 into 1eb1666b7c4d4b5fba5872d530baac3d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:18,876 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:18,876 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/C, priority=13, startTime=1731645678027; duration=0sec 2024-11-15T04:41:18,876 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:18,876 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:C 2024-11-15T04:41:18,887 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/d834aad3ccce48a8ae69408154411663 2024-11-15T04:41:18,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/8ceed50673bc467c9b5bec053b0172fb is 50, key is test_row_0/B:col10/1731645676453/Put/seqid=0 2024-11-15T04:41:18,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742207_1383 (size=12001) 2024-11-15T04:41:18,899 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/8ceed50673bc467c9b5bec053b0172fb 2024-11-15T04:41:18,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/e356541ac0b040bd998690f6a78fd47f is 50, key is test_row_0/C:col10/1731645676453/Put/seqid=0 2024-11-15T04:41:18,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742208_1384 (size=12001) 2024-11-15T04:41:18,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645738963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645738963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645738983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645738983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645739273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645739273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T04:41:19,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645739303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645739303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,319 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/e356541ac0b040bd998690f6a78fd47f 2024-11-15T04:41:19,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/d834aad3ccce48a8ae69408154411663 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d834aad3ccce48a8ae69408154411663 2024-11-15T04:41:19,326 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d834aad3ccce48a8ae69408154411663, entries=150, sequenceid=76, filesize=30.2 K 2024-11-15T04:41:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/8ceed50673bc467c9b5bec053b0172fb as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/8ceed50673bc467c9b5bec053b0172fb 2024-11-15T04:41:19,333 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/8ceed50673bc467c9b5bec053b0172fb, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T04:41:19,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/e356541ac0b040bd998690f6a78fd47f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/e356541ac0b040bd998690f6a78fd47f 2024-11-15T04:41:19,347 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/e356541ac0b040bd998690f6a78fd47f, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T04:41:19,349 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 05250181cbe972c6b3047f25758a3996 in 1286ms, sequenceid=76, compaction requested=false 2024-11-15T04:41:19,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:19,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:19,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-15T04:41:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-15T04:41:19,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-15T04:41:19,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2060 sec 2024-11-15T04:41:19,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.2120 sec 2024-11-15T04:41:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:19,785 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:19,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T04:41:19,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ff7e7a9ff86f4336b493d002677c6548_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645679785/Put/seqid=0 2024-11-15T04:41:19,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742209_1385 (size=17034) 2024-11-15T04:41:19,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645739842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645739843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645739843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645739854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645739963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645739963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645739963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:19,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645739972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:20,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645740174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:20,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645740183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:20,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645740185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:20,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645740193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:20,199 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:20,204 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ff7e7a9ff86f4336b493d002677c6548_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ff7e7a9ff86f4336b493d002677c6548_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:20,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/35e6444e94544dfcbb94c0ff825cb9d3, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:20,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/35e6444e94544dfcbb94c0ff825cb9d3 is 175, key is test_row_0/A:col10/1731645679785/Put/seqid=0 2024-11-15T04:41:20,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742210_1386 (size=48139) 2024-11-15T04:41:20,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645740493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:20,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:20,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645740502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:20,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645740502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:20,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645740502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:20,613 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/35e6444e94544dfcbb94c0ff825cb9d3 2024-11-15T04:41:20,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/26f28eb3dec24e0eb502b44647833bd3 is 50, key is test_row_0/B:col10/1731645679785/Put/seqid=0 2024-11-15T04:41:20,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742211_1387 (size=12001) 2024-11-15T04:41:20,637 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T04:41:20,637 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T04:41:21,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645741012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:21,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645741013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:21,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645741013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:21,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:21,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:21,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645741023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:21,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:21,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/26f28eb3dec24e0eb502b44647833bd3 2024-11-15T04:41:21,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/23b2f949cdd34c4abeab7a16542b50f9 is 50, key is test_row_0/C:col10/1731645679785/Put/seqid=0 2024-11-15T04:41:21,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742212_1388 (size=12001) 2024-11-15T04:41:21,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645741263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:21,273 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in 05250181cbe972c6b3047f25758a3996 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:41:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T04:41:21,303 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:21,303 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-15T04:41:21,305 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:21,306 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:21,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T04:41:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T04:41:21,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/23b2f949cdd34c4abeab7a16542b50f9 2024-11-15T04:41:21,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/35e6444e94544dfcbb94c0ff825cb9d3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/35e6444e94544dfcbb94c0ff825cb9d3 2024-11-15T04:41:21,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/35e6444e94544dfcbb94c0ff825cb9d3, entries=250, sequenceid=93, filesize=47.0 K 2024-11-15T04:41:21,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/26f28eb3dec24e0eb502b44647833bd3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/26f28eb3dec24e0eb502b44647833bd3 2024-11-15T04:41:21,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/26f28eb3dec24e0eb502b44647833bd3, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T04:41:21,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T04:41:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:21,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/23b2f949cdd34c4abeab7a16542b50f9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/23b2f949cdd34c4abeab7a16542b50f9 2024-11-15T04:41:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:21,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/23b2f949cdd34c4abeab7a16542b50f9, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T04:41:21,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 05250181cbe972c6b3047f25758a3996 in 1696ms, sequenceid=93, compaction requested=true 2024-11-15T04:41:21,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:21,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:21,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:21,481 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:21,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:21,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:21,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:21,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T04:41:21,481 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:21,483 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:21,483 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:21,483 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/A is initiating minor compaction (all files) 2024-11-15T04:41:21,483 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/B is initiating minor compaction (all files) 2024-11-15T04:41:21,483 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/A in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:21,483 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/B in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:21,483 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/4d218eb505334a03811b313cd379d359, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/8ceed50673bc467c9b5bec053b0172fb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/26f28eb3dec24e0eb502b44647833bd3] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=35.3 K 2024-11-15T04:41:21,483 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/60f05b03dfeb4f58999ddf3e4272e23c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d834aad3ccce48a8ae69408154411663, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/35e6444e94544dfcbb94c0ff825cb9d3] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=107.6 K 2024-11-15T04:41:21,483 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:21,483 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/60f05b03dfeb4f58999ddf3e4272e23c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d834aad3ccce48a8ae69408154411663, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/35e6444e94544dfcbb94c0ff825cb9d3] 2024-11-15T04:41:21,484 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d218eb505334a03811b313cd379d359, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645674189 2024-11-15T04:41:21,484 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60f05b03dfeb4f58999ddf3e4272e23c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645674189 2024-11-15T04:41:21,484 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ceed50673bc467c9b5bec053b0172fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731645676453 2024-11-15T04:41:21,484 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d834aad3ccce48a8ae69408154411663, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731645676453 2024-11-15T04:41:21,484 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 26f28eb3dec24e0eb502b44647833bd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645678641 2024-11-15T04:41:21,484 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35e6444e94544dfcbb94c0ff825cb9d3, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645678641 2024-11-15T04:41:21,499 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:21,512 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:21,513 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/198a28a8d50b46679a0aafcd6b03e9d3 is 50, key is test_row_0/B:col10/1731645679785/Put/seqid=0 2024-11-15T04:41:21,527 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411159291b20783bc45258b8e7d6d41018673_05250181cbe972c6b3047f25758a3996 store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:21,529 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411159291b20783bc45258b8e7d6d41018673_05250181cbe972c6b3047f25758a3996, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:21,529 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159291b20783bc45258b8e7d6d41018673_05250181cbe972c6b3047f25758a3996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:21,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742213_1389 (size=12207) 2024-11-15T04:41:21,571 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/198a28a8d50b46679a0aafcd6b03e9d3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/198a28a8d50b46679a0aafcd6b03e9d3 2024-11-15T04:41:21,578 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/B of 05250181cbe972c6b3047f25758a3996 into 198a28a8d50b46679a0aafcd6b03e9d3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:21,578 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:21,578 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/B, priority=13, startTime=1731645681481; duration=0sec 2024-11-15T04:41:21,578 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:21,578 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:B 2024-11-15T04:41:21,578 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:21,579 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:21,579 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/C is initiating minor compaction (all files) 2024-11-15T04:41:21,580 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/C in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:21,580 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/1eb1666b7c4d4b5fba5872d530baac3d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/e356541ac0b040bd998690f6a78fd47f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/23b2f949cdd34c4abeab7a16542b50f9] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=35.3 K 2024-11-15T04:41:21,580 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb1666b7c4d4b5fba5872d530baac3d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645674189 2024-11-15T04:41:21,580 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e356541ac0b040bd998690f6a78fd47f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731645676453 2024-11-15T04:41:21,581 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b2f949cdd34c4abeab7a16542b50f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645678641 2024-11-15T04:41:21,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742214_1390 (size=4469) 2024-11-15T04:41:21,590 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#A#compaction#329 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:21,590 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/d5da0077d86a4946a80c8e4289a27d9a is 175, key is test_row_0/A:col10/1731645679785/Put/seqid=0 2024-11-15T04:41:21,593 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#C#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:21,594 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/fee98083d8684a23b7f667c739a22b7c is 50, key is test_row_0/C:col10/1731645679785/Put/seqid=0 2024-11-15T04:41:21,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T04:41:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:21,614 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T04:41:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T04:41:21,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742215_1391 (size=31161) 2024-11-15T04:41:21,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742216_1392 (size=12207) 2024-11-15T04:41:21,662 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/fee98083d8684a23b7f667c739a22b7c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/fee98083d8684a23b7f667c739a22b7c 2024-11-15T04:41:21,667 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/C of 05250181cbe972c6b3047f25758a3996 into fee98083d8684a23b7f667c739a22b7c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:21,667 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:21,667 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/C, priority=13, startTime=1731645681481; duration=0sec 2024-11-15T04:41:21,667 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:21,667 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:C 2024-11-15T04:41:21,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111519de19c0411343db97493af5097c3968_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645679839/Put/seqid=0 2024-11-15T04:41:21,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742217_1393 (size=12154) 2024-11-15T04:41:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T04:41:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:22,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:22,045 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/d5da0077d86a4946a80c8e4289a27d9a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d5da0077d86a4946a80c8e4289a27d9a 2024-11-15T04:41:22,054 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/A of 05250181cbe972c6b3047f25758a3996 into d5da0077d86a4946a80c8e4289a27d9a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:22,054 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:22,054 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/A, priority=13, startTime=1731645681481; duration=0sec 2024-11-15T04:41:22,054 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:22,054 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:A 2024-11-15T04:41:22,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645742079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645742079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645742081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645742081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,098 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111519de19c0411343db97493af5097c3968_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111519de19c0411343db97493af5097c3968_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:22,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/d40ae2a9cb82402c802d7691bca29225, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:22,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/d40ae2a9cb82402c802d7691bca29225 is 175, key is test_row_0/A:col10/1731645679839/Put/seqid=0 2024-11-15T04:41:22,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742218_1394 (size=30955) 2024-11-15T04:41:22,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645742203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645742204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645742204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645742204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645742422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645742423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645742423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645742423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T04:41:22,560 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/d40ae2a9cb82402c802d7691bca29225 2024-11-15T04:41:22,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/171dd355621b461b9aec68f8286f2200 is 50, key is test_row_0/B:col10/1731645679839/Put/seqid=0 2024-11-15T04:41:22,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742219_1395 (size=12001) 2024-11-15T04:41:22,640 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/171dd355621b461b9aec68f8286f2200 2024-11-15T04:41:22,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/70379ebc4dd9440a9a8d2682b659ca54 is 50, key is test_row_0/C:col10/1731645679839/Put/seqid=0 2024-11-15T04:41:22,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742220_1396 (size=12001) 2024-11-15T04:41:22,717 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/70379ebc4dd9440a9a8d2682b659ca54 2024-11-15T04:41:22,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/d40ae2a9cb82402c802d7691bca29225 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d40ae2a9cb82402c802d7691bca29225 2024-11-15T04:41:22,729 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d40ae2a9cb82402c802d7691bca29225, entries=150, sequenceid=116, filesize=30.2 K 2024-11-15T04:41:22,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/171dd355621b461b9aec68f8286f2200 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/171dd355621b461b9aec68f8286f2200 2024-11-15T04:41:22,734 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/171dd355621b461b9aec68f8286f2200, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T04:41:22,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/70379ebc4dd9440a9a8d2682b659ca54 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/70379ebc4dd9440a9a8d2682b659ca54 2024-11-15T04:41:22,742 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/70379ebc4dd9440a9a8d2682b659ca54, entries=150, sequenceid=116, filesize=11.7 K 2024-11-15T04:41:22,744 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 05250181cbe972c6b3047f25758a3996 in 1131ms, sequenceid=116, compaction requested=false 2024-11-15T04:41:22,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:22,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:22,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-15T04:41:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-15T04:41:22,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-15T04:41:22,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4390 sec 2024-11-15T04:41:22,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4460 sec 2024-11-15T04:41:22,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:22,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T04:41:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:22,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fa088d7faeab4ea1822aa4b5bcea8db4_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645682754/Put/seqid=0 2024-11-15T04:41:22,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742221_1397 (size=12304) 2024-11-15T04:41:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645742807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645742807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645742808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645742811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645742933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645742933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645742933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:22,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645742933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:23,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645743143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:23,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645743143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:23,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645743153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:23,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645743153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:23,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:23,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fa088d7faeab4ea1822aa4b5bcea8db4_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fa088d7faeab4ea1822aa4b5bcea8db4_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:23,186 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/f204fd5d66ab4db99187a2832deacadd, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:23,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/f204fd5d66ab4db99187a2832deacadd is 175, key is test_row_0/A:col10/1731645682754/Put/seqid=0 2024-11-15T04:41:23,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742222_1398 (size=31105) 2024-11-15T04:41:23,233 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/f204fd5d66ab4db99187a2832deacadd 2024-11-15T04:41:23,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/897f081806654b349576f858edb40609 is 50, key is test_row_0/B:col10/1731645682754/Put/seqid=0 2024-11-15T04:41:23,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742223_1399 (size=12151) 2024-11-15T04:41:23,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/897f081806654b349576f858edb40609 2024-11-15T04:41:23,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/05165d6af54d4507b1af21895d008859 is 50, key is test_row_0/C:col10/1731645682754/Put/seqid=0 2024-11-15T04:41:23,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742224_1400 (size=12151) 2024-11-15T04:41:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T04:41:23,453 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:23,453 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-15T04:41:23,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T04:41:23,455 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:23,456 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:23,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:23,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645743464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:23,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645743465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645743465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:23,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:23,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645743473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:23,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:23,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T04:41:23,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T04:41:23,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:23,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:23,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:23,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:23,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:23,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/05165d6af54d4507b1af21895d008859 2024-11-15T04:41:23,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/f204fd5d66ab4db99187a2832deacadd as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/f204fd5d66ab4db99187a2832deacadd 2024-11-15T04:41:23,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/f204fd5d66ab4db99187a2832deacadd, entries=150, sequenceid=135, filesize=30.4 K 2024-11-15T04:41:23,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/897f081806654b349576f858edb40609 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/897f081806654b349576f858edb40609 2024-11-15T04:41:23,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/897f081806654b349576f858edb40609, entries=150, sequenceid=135, filesize=11.9 K 2024-11-15T04:41:23,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/05165d6af54d4507b1af21895d008859 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/05165d6af54d4507b1af21895d008859 2024-11-15T04:41:23,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/05165d6af54d4507b1af21895d008859, entries=150, sequenceid=135, filesize=11.9 K 2024-11-15T04:41:23,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 05250181cbe972c6b3047f25758a3996 in 953ms, sequenceid=135, compaction requested=true 2024-11-15T04:41:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:23,709 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:23,709 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:23,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:23,710 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:23,710 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/A is initiating minor compaction (all files) 2024-11-15T04:41:23,710 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/A in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:23,710 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d5da0077d86a4946a80c8e4289a27d9a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d40ae2a9cb82402c802d7691bca29225, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/f204fd5d66ab4db99187a2832deacadd] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=91.0 K 2024-11-15T04:41:23,710 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:23,710 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d5da0077d86a4946a80c8e4289a27d9a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d40ae2a9cb82402c802d7691bca29225, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/f204fd5d66ab4db99187a2832deacadd] 2024-11-15T04:41:23,710 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:23,710 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/B is initiating minor compaction (all files) 2024-11-15T04:41:23,710 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/B in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:23,710 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/198a28a8d50b46679a0aafcd6b03e9d3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/171dd355621b461b9aec68f8286f2200, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/897f081806654b349576f858edb40609] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=35.5 K 2024-11-15T04:41:23,710 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5da0077d86a4946a80c8e4289a27d9a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645678641 2024-11-15T04:41:23,710 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 198a28a8d50b46679a0aafcd6b03e9d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645678641 2024-11-15T04:41:23,711 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d40ae2a9cb82402c802d7691bca29225, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731645679839 2024-11-15T04:41:23,711 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 171dd355621b461b9aec68f8286f2200, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731645679839 2024-11-15T04:41:23,711 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting f204fd5d66ab4db99187a2832deacadd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731645682077 2024-11-15T04:41:23,711 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 897f081806654b349576f858edb40609, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731645682077 2024-11-15T04:41:23,719 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#B#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:23,719 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/9ac76a17c0c44eeda8a43eaccb03c661 is 50, key is test_row_0/B:col10/1731645682754/Put/seqid=0 2024-11-15T04:41:23,722 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:23,729 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115135be712870b4b8dab7ce2674a1e822a_05250181cbe972c6b3047f25758a3996 store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:23,731 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115135be712870b4b8dab7ce2674a1e822a_05250181cbe972c6b3047f25758a3996, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:23,731 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115135be712870b4b8dab7ce2674a1e822a_05250181cbe972c6b3047f25758a3996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:23,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742225_1401 (size=12459) 2024-11-15T04:41:23,742 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/9ac76a17c0c44eeda8a43eaccb03c661 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/9ac76a17c0c44eeda8a43eaccb03c661 2024-11-15T04:41:23,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742226_1402 (size=4469) 2024-11-15T04:41:23,746 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#A#compaction#339 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:23,747 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/0d7c10ef15014a0fbfadf0264c0dcd9f is 175, key is test_row_0/A:col10/1731645682754/Put/seqid=0 2024-11-15T04:41:23,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742227_1403 (size=31413) 2024-11-15T04:41:23,752 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/B of 05250181cbe972c6b3047f25758a3996 into 9ac76a17c0c44eeda8a43eaccb03c661(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:23,752 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:23,752 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/B, priority=13, startTime=1731645683709; duration=0sec 2024-11-15T04:41:23,752 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:23,752 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:B 2024-11-15T04:41:23,752 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:23,753 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:23,753 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/C is initiating minor compaction (all files) 2024-11-15T04:41:23,753 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/C in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:23,753 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/fee98083d8684a23b7f667c739a22b7c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/70379ebc4dd9440a9a8d2682b659ca54, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/05165d6af54d4507b1af21895d008859] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=35.5 K 2024-11-15T04:41:23,753 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting fee98083d8684a23b7f667c739a22b7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731645678641 2024-11-15T04:41:23,754 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 70379ebc4dd9440a9a8d2682b659ca54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731645679839 2024-11-15T04:41:23,754 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 05165d6af54d4507b1af21895d008859, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731645682077 2024-11-15T04:41:23,756 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/0d7c10ef15014a0fbfadf0264c0dcd9f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/0d7c10ef15014a0fbfadf0264c0dcd9f 2024-11-15T04:41:23,760 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#C#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:23,761 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/26371d7a9b3c4edf8f53a0e9ba546c59 is 50, key is test_row_0/C:col10/1731645682754/Put/seqid=0 2024-11-15T04:41:23,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T04:41:23,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:23,762 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T04:41:23,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742228_1404 (size=12459) 2024-11-15T04:41:23,771 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/A of 05250181cbe972c6b3047f25758a3996 into 0d7c10ef15014a0fbfadf0264c0dcd9f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:23,772 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:23,772 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/A, priority=13, startTime=1731645683709; duration=0sec 2024-11-15T04:41:23,772 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:23,772 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:A 2024-11-15T04:41:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T04:41:23,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156da81b86e6bb4d7e9cfc9f2f3f6e7b7d_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645682805/Put/seqid=0 2024-11-15T04:41:23,781 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/26371d7a9b3c4edf8f53a0e9ba546c59 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/26371d7a9b3c4edf8f53a0e9ba546c59 2024-11-15T04:41:23,788 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/C of 05250181cbe972c6b3047f25758a3996 into 26371d7a9b3c4edf8f53a0e9ba546c59(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:23,788 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:23,788 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/C, priority=13, startTime=1731645683709; duration=0sec 2024-11-15T04:41:23,788 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:23,788 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:C 2024-11-15T04:41:23,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742229_1405 (size=12304) 2024-11-15T04:41:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:23,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:24,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645744012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645744014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645744020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645744021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T04:41:24,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645744133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645744133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645744143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645744143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:24,203 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156da81b86e6bb4d7e9cfc9f2f3f6e7b7d_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156da81b86e6bb4d7e9cfc9f2f3f6e7b7d_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:24,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/a311ad2e9a494a14b35da2f3fcd91213, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:24,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/a311ad2e9a494a14b35da2f3fcd91213 is 175, key is test_row_0/A:col10/1731645682805/Put/seqid=0 2024-11-15T04:41:24,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742230_1406 (size=31105) 2024-11-15T04:41:24,234 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/a311ad2e9a494a14b35da2f3fcd91213 2024-11-15T04:41:24,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/df5d427906504ef9a464578cdc36addd is 50, key is test_row_0/B:col10/1731645682805/Put/seqid=0 2024-11-15T04:41:24,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742231_1407 (size=12151) 2024-11-15T04:41:24,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645744355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645744355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645744362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645744362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T04:41:24,658 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/df5d427906504ef9a464578cdc36addd 2024-11-15T04:41:24,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/38c2e0020b4d47b8befdedb4044fdd5f is 50, key is test_row_0/C:col10/1731645682805/Put/seqid=0 2024-11-15T04:41:24,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742232_1408 (size=12151) 2024-11-15T04:41:24,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645744672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645744672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645744673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645744673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:24,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,069 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/38c2e0020b4d47b8befdedb4044fdd5f 2024-11-15T04:41:25,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/a311ad2e9a494a14b35da2f3fcd91213 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/a311ad2e9a494a14b35da2f3fcd91213 2024-11-15T04:41:25,078 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/a311ad2e9a494a14b35da2f3fcd91213, entries=150, sequenceid=155, filesize=30.4 K 2024-11-15T04:41:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/df5d427906504ef9a464578cdc36addd as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/df5d427906504ef9a464578cdc36addd 2024-11-15T04:41:25,086 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/df5d427906504ef9a464578cdc36addd, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T04:41:25,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/38c2e0020b4d47b8befdedb4044fdd5f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/38c2e0020b4d47b8befdedb4044fdd5f 2024-11-15T04:41:25,093 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/38c2e0020b4d47b8befdedb4044fdd5f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T04:41:25,094 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 05250181cbe972c6b3047f25758a3996 in 1332ms, sequenceid=155, compaction requested=false 2024-11-15T04:41:25,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:25,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:25,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-15T04:41:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-15T04:41:25,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-15T04:41:25,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6400 sec 2024-11-15T04:41:25,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.6450 sec 2024-11-15T04:41:25,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:25,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:25,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T04:41:25,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b3225c15722b42feacd914f2aaf25b5a_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645684003/Put/seqid=0 2024-11-15T04:41:25,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742233_1409 (size=12304) 2024-11-15T04:41:25,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645745224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645745233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645745237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645745237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645745343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645745343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645745352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645745352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645745563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645745563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645745572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645745572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T04:41:25,615 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:25,615 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-15T04:41:25,618 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T04:41:25,619 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:25,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:25,625 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:25,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b3225c15722b42feacd914f2aaf25b5a_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b3225c15722b42feacd914f2aaf25b5a_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:25,633 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/39468c9d75be47ccb86297575218f61f, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:25,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/39468c9d75be47ccb86297575218f61f is 175, key is test_row_0/A:col10/1731645684003/Put/seqid=0 2024-11-15T04:41:25,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742234_1410 (size=31105) 2024-11-15T04:41:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T04:41:25,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T04:41:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:25,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:25,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645745882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645745883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645745893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645745893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:25,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T04:41:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:25,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:25,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T04:41:26,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/39468c9d75be47ccb86297575218f61f 2024-11-15T04:41:26,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T04:41:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:26,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:26,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/15a5e2e4c2844f699edb4d4c00b8aeca is 50, key is test_row_0/B:col10/1731645684003/Put/seqid=0 2024-11-15T04:41:26,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742235_1411 (size=12151) 2024-11-15T04:41:26,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/15a5e2e4c2844f699edb4d4c00b8aeca 2024-11-15T04:41:26,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/9c6c2f21cb0c48388e8b3eeaf32d6276 is 50, key is test_row_0/C:col10/1731645684003/Put/seqid=0 2024-11-15T04:41:26,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742236_1412 (size=12151) 2024-11-15T04:41:26,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/9c6c2f21cb0c48388e8b3eeaf32d6276 2024-11-15T04:41:26,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/39468c9d75be47ccb86297575218f61f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/39468c9d75be47ccb86297575218f61f 2024-11-15T04:41:26,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/39468c9d75be47ccb86297575218f61f, entries=150, sequenceid=175, filesize=30.4 K 2024-11-15T04:41:26,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/15a5e2e4c2844f699edb4d4c00b8aeca as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/15a5e2e4c2844f699edb4d4c00b8aeca 2024-11-15T04:41:26,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/15a5e2e4c2844f699edb4d4c00b8aeca, entries=150, sequenceid=175, filesize=11.9 K 2024-11-15T04:41:26,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/9c6c2f21cb0c48388e8b3eeaf32d6276 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/9c6c2f21cb0c48388e8b3eeaf32d6276 2024-11-15T04:41:26,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T04:41:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:26,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:26,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:26,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:26,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:26,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/9c6c2f21cb0c48388e8b3eeaf32d6276, entries=150, sequenceid=175, filesize=11.9 K 2024-11-15T04:41:26,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 05250181cbe972c6b3047f25758a3996 in 1049ms, sequenceid=175, compaction requested=true 2024-11-15T04:41:26,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:26,239 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:26,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:26,240 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:26,240 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/A is initiating minor compaction (all files) 2024-11-15T04:41:26,240 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/A in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:26,240 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/0d7c10ef15014a0fbfadf0264c0dcd9f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/a311ad2e9a494a14b35da2f3fcd91213, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/39468c9d75be47ccb86297575218f61f] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=91.4 K 2024-11-15T04:41:26,240 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:26,240 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/0d7c10ef15014a0fbfadf0264c0dcd9f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/a311ad2e9a494a14b35da2f3fcd91213, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/39468c9d75be47ccb86297575218f61f] 2024-11-15T04:41:26,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:26,240 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d7c10ef15014a0fbfadf0264c0dcd9f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731645682077 2024-11-15T04:41:26,240 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:26,241 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting a311ad2e9a494a14b35da2f3fcd91213, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731645682805 2024-11-15T04:41:26,241 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39468c9d75be47ccb86297575218f61f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731645684003 2024-11-15T04:41:26,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:26,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:26,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:26,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:26,241 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:26,241 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/B is initiating minor compaction (all files) 2024-11-15T04:41:26,241 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/B in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:26,242 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/9ac76a17c0c44eeda8a43eaccb03c661, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/df5d427906504ef9a464578cdc36addd, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/15a5e2e4c2844f699edb4d4c00b8aeca] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=35.9 K 2024-11-15T04:41:26,242 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac76a17c0c44eeda8a43eaccb03c661, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731645682077 2024-11-15T04:41:26,242 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting df5d427906504ef9a464578cdc36addd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731645682805 2024-11-15T04:41:26,242 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 15a5e2e4c2844f699edb4d4c00b8aeca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731645684003 2024-11-15T04:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T04:41:26,248 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:26,256 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#B#compaction#348 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:26,256 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/bdade7e0d0ab4f2899607c567fb6bdee is 50, key is test_row_0/B:col10/1731645684003/Put/seqid=0 2024-11-15T04:41:26,259 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115816f90e030fa44f7b0aad9eb249fed52_05250181cbe972c6b3047f25758a3996 store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:26,261 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115816f90e030fa44f7b0aad9eb249fed52_05250181cbe972c6b3047f25758a3996, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:26,261 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115816f90e030fa44f7b0aad9eb249fed52_05250181cbe972c6b3047f25758a3996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:26,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742237_1413 (size=12561) 2024-11-15T04:41:26,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742238_1414 (size=4469) 2024-11-15T04:41:26,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T04:41:26,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:26,386 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T04:41:26,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150648ca5b4f47445486ba66ee11e64f0a_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645685235/Put/seqid=0 2024-11-15T04:41:26,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742239_1415 (size=12304) 2024-11-15T04:41:26,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:26,404 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150648ca5b4f47445486ba66ee11e64f0a_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150648ca5b4f47445486ba66ee11e64f0a_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:26,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:26,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/54f9e805cd5a43cabaa50eadad6d49ab, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:26,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/54f9e805cd5a43cabaa50eadad6d49ab is 175, key is test_row_0/A:col10/1731645685235/Put/seqid=0 2024-11-15T04:41:26,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742240_1416 (size=31105) 2024-11-15T04:41:26,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645746440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645746442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645746450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645746451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645746562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645746562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645746562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645746573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,670 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/bdade7e0d0ab4f2899607c567fb6bdee as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/bdade7e0d0ab4f2899607c567fb6bdee 2024-11-15T04:41:26,675 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/B of 05250181cbe972c6b3047f25758a3996 into bdade7e0d0ab4f2899607c567fb6bdee(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:26,675 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:26,675 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/B, priority=13, startTime=1731645686240; duration=0sec 2024-11-15T04:41:26,675 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:26,675 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:B 2024-11-15T04:41:26,675 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:26,676 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:26,676 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/C is initiating minor compaction (all files) 2024-11-15T04:41:26,676 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/C in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:26,676 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/26371d7a9b3c4edf8f53a0e9ba546c59, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/38c2e0020b4d47b8befdedb4044fdd5f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/9c6c2f21cb0c48388e8b3eeaf32d6276] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=35.9 K 2024-11-15T04:41:26,679 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 26371d7a9b3c4edf8f53a0e9ba546c59, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731645682077 2024-11-15T04:41:26,679 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#A#compaction#347 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:26,679 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c2e0020b4d47b8befdedb4044fdd5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731645682805 2024-11-15T04:41:26,679 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/db5652c09ac941fcb8d26961a13d7d29 is 175, key is test_row_0/A:col10/1731645684003/Put/seqid=0 2024-11-15T04:41:26,680 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c6c2f21cb0c48388e8b3eeaf32d6276, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731645684003 2024-11-15T04:41:26,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742241_1417 (size=31515) 2024-11-15T04:41:26,697 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:26,697 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/66e3eeb6f4604f379bae8db991af06e7 is 50, key is test_row_0/C:col10/1731645684003/Put/seqid=0 2024-11-15T04:41:26,703 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/db5652c09ac941fcb8d26961a13d7d29 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/db5652c09ac941fcb8d26961a13d7d29 2024-11-15T04:41:26,710 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/A of 05250181cbe972c6b3047f25758a3996 into db5652c09ac941fcb8d26961a13d7d29(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:26,710 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:26,710 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/A, priority=13, startTime=1731645686239; duration=0sec 2024-11-15T04:41:26,710 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:26,710 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:A 2024-11-15T04:41:26,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742242_1418 (size=12561) 2024-11-15T04:41:26,722 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/66e3eeb6f4604f379bae8db991af06e7 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/66e3eeb6f4604f379bae8db991af06e7 2024-11-15T04:41:26,726 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/C of 05250181cbe972c6b3047f25758a3996 into 66e3eeb6f4604f379bae8db991af06e7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:26,726 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:26,726 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/C, priority=13, startTime=1731645686241; duration=0sec 2024-11-15T04:41:26,727 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:26,727 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:C 2024-11-15T04:41:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T04:41:26,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645746782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645746783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645746783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645746783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:26,812 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/54f9e805cd5a43cabaa50eadad6d49ab 2024-11-15T04:41:26,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/6dbc7de3f2f9429aa000644e91c4c763 is 50, key is test_row_0/B:col10/1731645685235/Put/seqid=0 2024-11-15T04:41:26,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742243_1419 (size=12151) 2024-11-15T04:41:27,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:27,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645747103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:27,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645747103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:27,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645747103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:27,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645747104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:27,244 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/6dbc7de3f2f9429aa000644e91c4c763 2024-11-15T04:41:27,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/f8da1688c3ea44ae8cf2f6fd4e3c71df is 50, key is test_row_0/C:col10/1731645685235/Put/seqid=0 2024-11-15T04:41:27,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742244_1420 (size=12151) 2024-11-15T04:41:27,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645747613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:27,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645747623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:27,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645747623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645747623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:27,665 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/f8da1688c3ea44ae8cf2f6fd4e3c71df 2024-11-15T04:41:27,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/54f9e805cd5a43cabaa50eadad6d49ab as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/54f9e805cd5a43cabaa50eadad6d49ab 2024-11-15T04:41:27,672 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/54f9e805cd5a43cabaa50eadad6d49ab, entries=150, sequenceid=194, filesize=30.4 K 2024-11-15T04:41:27,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/6dbc7de3f2f9429aa000644e91c4c763 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/6dbc7de3f2f9429aa000644e91c4c763 2024-11-15T04:41:27,676 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/6dbc7de3f2f9429aa000644e91c4c763, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T04:41:27,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/f8da1688c3ea44ae8cf2f6fd4e3c71df as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/f8da1688c3ea44ae8cf2f6fd4e3c71df 2024-11-15T04:41:27,680 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/f8da1688c3ea44ae8cf2f6fd4e3c71df, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T04:41:27,680 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 05250181cbe972c6b3047f25758a3996 in 1294ms, sequenceid=194, compaction requested=false 2024-11-15T04:41:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-15T04:41:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-15T04:41:27,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-15T04:41:27,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0620 sec 2024-11-15T04:41:27,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0690 sec 2024-11-15T04:41:27,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T04:41:27,763 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:27,763 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-15T04:41:27,765 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T04:41:27,765 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:27,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T04:41:27,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T04:41:27,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:27,918 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T04:41:27,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c742d2d4bcd44eaeb617495324eec2c2_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645686449/Put/seqid=0 2024-11-15T04:41:27,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742245_1421 (size=12304) 2024-11-15T04:41:27,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:27,930 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c742d2d4bcd44eaeb617495324eec2c2_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c742d2d4bcd44eaeb617495324eec2c2_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/81ac4e0f7ec24730b9f114193861d051, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/81ac4e0f7ec24730b9f114193861d051 is 175, key is test_row_0/A:col10/1731645686449/Put/seqid=0 2024-11-15T04:41:27,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742246_1422 (size=31105) 2024-11-15T04:41:28,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T04:41:28,336 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/81ac4e0f7ec24730b9f114193861d051 2024-11-15T04:41:28,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/7f8906879c514c9e8f7471b5ac650014 is 50, key is test_row_0/B:col10/1731645686449/Put/seqid=0 2024-11-15T04:41:28,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742247_1423 (size=12151) 2024-11-15T04:41:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T04:41:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:28,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:28,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645748665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645748666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645748667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:28,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645748676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:28,751 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/7f8906879c514c9e8f7471b5ac650014 2024-11-15T04:41:28,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/3964387923b6432aa6ee88747f55a9a6 is 50, key is test_row_0/C:col10/1731645686449/Put/seqid=0 2024-11-15T04:41:28,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742248_1424 (size=12151) 2024-11-15T04:41:28,777 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/3964387923b6432aa6ee88747f55a9a6 2024-11-15T04:41:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645748784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:28,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/81ac4e0f7ec24730b9f114193861d051 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/81ac4e0f7ec24730b9f114193861d051 2024-11-15T04:41:28,790 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/81ac4e0f7ec24730b9f114193861d051, entries=150, sequenceid=214, filesize=30.4 K 2024-11-15T04:41:28,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/7f8906879c514c9e8f7471b5ac650014 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/7f8906879c514c9e8f7471b5ac650014 2024-11-15T04:41:28,796 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/7f8906879c514c9e8f7471b5ac650014, entries=150, sequenceid=214, filesize=11.9 K 2024-11-15T04:41:28,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645748784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645748785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645748793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/3964387923b6432aa6ee88747f55a9a6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/3964387923b6432aa6ee88747f55a9a6 2024-11-15T04:41:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:28,803 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/3964387923b6432aa6ee88747f55a9a6, entries=150, sequenceid=214, filesize=11.9 K 2024-11-15T04:41:28,804 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 05250181cbe972c6b3047f25758a3996 in 886ms, sequenceid=214, compaction requested=true 2024-11-15T04:41:28,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:28,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:28,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-15T04:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-15T04:41:28,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-15T04:41:28,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0400 sec 2024-11-15T04:41:28,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.0470 sec 2024-11-15T04:41:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T04:41:28,903 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:28,904 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-15T04:41:28,905 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T04:41:28,906 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:28,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:28,996 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:28,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T04:41:29,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111506340d2a58894176838d5f0e02b32532_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645688994/Put/seqid=0 2024-11-15T04:41:29,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T04:41:29,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742249_1425 (size=14794) 2024-11-15T04:41:29,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645749035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645749046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645749047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T04:41:29,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:29,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645749048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645749152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645749152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645749152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645749163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T04:41:29,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:29,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T04:41:29,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T04:41:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645749372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645749372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645749372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645749383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,417 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:29,419 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111506340d2a58894176838d5f0e02b32532_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111506340d2a58894176838d5f0e02b32532_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:29,420 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/8dc0de908a2642ffa8866b47a10441c8, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:29,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/8dc0de908a2642ffa8866b47a10441c8 is 175, key is test_row_0/A:col10/1731645688994/Put/seqid=0 2024-11-15T04:41:29,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742250_1426 (size=39749) 2024-11-15T04:41:29,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T04:41:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T04:41:29,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T04:41:29,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:29,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645749692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645749692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645749693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645749693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:29,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:29,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T04:41:29,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:29,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,824 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/8dc0de908a2642ffa8866b47a10441c8 2024-11-15T04:41:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:29,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/db9da9b0231f49eebed955f86c2184cb is 50, key is test_row_0/B:col10/1731645688994/Put/seqid=0 2024-11-15T04:41:29,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742251_1427 (size=12151) 2024-11-15T04:41:29,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/db9da9b0231f49eebed955f86c2184cb 2024-11-15T04:41:29,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/ef83ed4325d4461fb3c61055457dd3c1 is 50, key is test_row_0/C:col10/1731645688994/Put/seqid=0 2024-11-15T04:41:29,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742252_1428 (size=12151) 2024-11-15T04:41:29,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/ef83ed4325d4461fb3c61055457dd3c1 2024-11-15T04:41:29,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/8dc0de908a2642ffa8866b47a10441c8 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8dc0de908a2642ffa8866b47a10441c8 2024-11-15T04:41:29,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8dc0de908a2642ffa8866b47a10441c8, entries=200, sequenceid=234, filesize=38.8 K 2024-11-15T04:41:29,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/db9da9b0231f49eebed955f86c2184cb as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/db9da9b0231f49eebed955f86c2184cb 2024-11-15T04:41:29,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/db9da9b0231f49eebed955f86c2184cb, entries=150, sequenceid=234, filesize=11.9 K 2024-11-15T04:41:29,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/ef83ed4325d4461fb3c61055457dd3c1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/ef83ed4325d4461fb3c61055457dd3c1 2024-11-15T04:41:29,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/ef83ed4325d4461fb3c61055457dd3c1, entries=150, sequenceid=234, filesize=11.9 K 2024-11-15T04:41:29,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 05250181cbe972c6b3047f25758a3996 in 881ms, sequenceid=234, compaction requested=true 2024-11-15T04:41:29,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:29,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:29,878 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:29,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:29,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:29,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:29,878 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:29,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:29,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:29,879 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:29,879 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/A is initiating minor compaction (all files) 2024-11-15T04:41:29,879 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/A in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,879 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/db5652c09ac941fcb8d26961a13d7d29, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/54f9e805cd5a43cabaa50eadad6d49ab, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/81ac4e0f7ec24730b9f114193861d051, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8dc0de908a2642ffa8866b47a10441c8] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=130.3 K 2024-11-15T04:41:29,879 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,879 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/db5652c09ac941fcb8d26961a13d7d29, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/54f9e805cd5a43cabaa50eadad6d49ab, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/81ac4e0f7ec24730b9f114193861d051, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8dc0de908a2642ffa8866b47a10441c8] 2024-11-15T04:41:29,880 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting db5652c09ac941fcb8d26961a13d7d29, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731645684003 2024-11-15T04:41:29,880 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f9e805cd5a43cabaa50eadad6d49ab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731645685208 2024-11-15T04:41:29,880 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ac4e0f7ec24730b9f114193861d051, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731645686439 2024-11-15T04:41:29,881 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dc0de908a2642ffa8866b47a10441c8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731645688664 2024-11-15T04:41:29,881 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:29,881 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/B is initiating minor compaction (all files) 2024-11-15T04:41:29,881 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/B in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,881 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/bdade7e0d0ab4f2899607c567fb6bdee, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/6dbc7de3f2f9429aa000644e91c4c763, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/7f8906879c514c9e8f7471b5ac650014, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/db9da9b0231f49eebed955f86c2184cb] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=47.9 K 2024-11-15T04:41:29,881 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting bdade7e0d0ab4f2899607c567fb6bdee, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731645684003 2024-11-15T04:41:29,881 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dbc7de3f2f9429aa000644e91c4c763, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731645685208 2024-11-15T04:41:29,882 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8906879c514c9e8f7471b5ac650014, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731645686439 2024-11-15T04:41:29,882 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting db9da9b0231f49eebed955f86c2184cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731645688664 2024-11-15T04:41:29,886 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:29,901 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#B#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:29,901 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/4044d057e12b456d8c0766aac6058a90 is 50, key is test_row_0/B:col10/1731645688994/Put/seqid=0 2024-11-15T04:41:29,911 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115abcf410f6b1e4607ae12fad0c2c1c272_05250181cbe972c6b3047f25758a3996 store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:29,914 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115abcf410f6b1e4607ae12fad0c2c1c272_05250181cbe972c6b3047f25758a3996, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:29,914 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115abcf410f6b1e4607ae12fad0c2c1c272_05250181cbe972c6b3047f25758a3996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:29,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742253_1429 (size=12697) 2024-11-15T04:41:29,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742254_1430 (size=4469) 2024-11-15T04:41:29,961 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#A#compaction#359 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:29,962 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/aa726dd3643240269de60459eb44e1be is 175, key is test_row_0/A:col10/1731645688994/Put/seqid=0 2024-11-15T04:41:29,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T04:41:29,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:29,979 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T04:41:29,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151a29a63b29594609873a7ed64f6b49e1_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645689028/Put/seqid=0 2024-11-15T04:41:29,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742255_1431 (size=31651) 2024-11-15T04:41:30,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742256_1432 (size=12304) 2024-11-15T04:41:30,001 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/aa726dd3643240269de60459eb44e1be as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/aa726dd3643240269de60459eb44e1be 2024-11-15T04:41:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:30,005 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151a29a63b29594609873a7ed64f6b49e1_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151a29a63b29594609873a7ed64f6b49e1_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:30,007 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 05250181cbe972c6b3047f25758a3996/A of 05250181cbe972c6b3047f25758a3996 into aa726dd3643240269de60459eb44e1be(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:30,007 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:30,007 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/A, priority=12, startTime=1731645689878; duration=0sec 2024-11-15T04:41:30,008 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:30,008 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:A 2024-11-15T04:41:30,008 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:30,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/1658774184c4498991002cbb55f1714f, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:30,010 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:30,010 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/C is initiating minor compaction (all files) 2024-11-15T04:41:30,010 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/C in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:30,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/1658774184c4498991002cbb55f1714f is 175, key is test_row_0/A:col10/1731645689028/Put/seqid=0 2024-11-15T04:41:30,010 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/66e3eeb6f4604f379bae8db991af06e7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/f8da1688c3ea44ae8cf2f6fd4e3c71df, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/3964387923b6432aa6ee88747f55a9a6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/ef83ed4325d4461fb3c61055457dd3c1] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=47.9 K 2024-11-15T04:41:30,010 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66e3eeb6f4604f379bae8db991af06e7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731645684003 2024-11-15T04:41:30,011 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8da1688c3ea44ae8cf2f6fd4e3c71df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731645685208 2024-11-15T04:41:30,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742257_1433 (size=31105) 2024-11-15T04:41:30,015 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3964387923b6432aa6ee88747f55a9a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731645686439 2024-11-15T04:41:30,015 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef83ed4325d4461fb3c61055457dd3c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731645688664 2024-11-15T04:41:30,035 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:30,035 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/58d30403ad044893ac97b104a5277a31 is 50, key is test_row_0/C:col10/1731645688994/Put/seqid=0 2024-11-15T04:41:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T04:41:30,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742258_1434 (size=12697) 2024-11-15T04:41:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:30,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. as already flushing 2024-11-15T04:41:30,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645750239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645750247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645750249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645750252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,343 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/4044d057e12b456d8c0766aac6058a90 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/4044d057e12b456d8c0766aac6058a90 2024-11-15T04:41:30,347 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 05250181cbe972c6b3047f25758a3996/B of 05250181cbe972c6b3047f25758a3996 into 4044d057e12b456d8c0766aac6058a90(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:30,347 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:30,347 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/B, priority=12, startTime=1731645689878; duration=0sec 2024-11-15T04:41:30,347 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:30,347 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:B 2024-11-15T04:41:30,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645750363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645750364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645750364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645750373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,415 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/1658774184c4498991002cbb55f1714f 2024-11-15T04:41:30,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/3a88d42276cc4af1b86c3b2ca24d8058 is 50, key is test_row_0/B:col10/1731645689028/Put/seqid=0 2024-11-15T04:41:30,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742259_1435 (size=12151) 2024-11-15T04:41:30,454 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/58d30403ad044893ac97b104a5277a31 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/58d30403ad044893ac97b104a5277a31 2024-11-15T04:41:30,458 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 05250181cbe972c6b3047f25758a3996/C of 05250181cbe972c6b3047f25758a3996 into 58d30403ad044893ac97b104a5277a31(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:30,458 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:30,458 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/C, priority=12, startTime=1731645689878; duration=0sec 2024-11-15T04:41:30,458 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:30,458 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:C 2024-11-15T04:41:30,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645750583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645750583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645750583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645750583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,824 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/3a88d42276cc4af1b86c3b2ca24d8058 2024-11-15T04:41:30,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/a5a552f5dae1481482563fcabda1e152 is 50, key is test_row_0/C:col10/1731645689028/Put/seqid=0 2024-11-15T04:41:30,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742260_1436 (size=12151) 2024-11-15T04:41:30,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645750903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645750903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645750904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:30,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645750904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:30,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T04:41:31,259 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/a5a552f5dae1481482563fcabda1e152 2024-11-15T04:41:31,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/1658774184c4498991002cbb55f1714f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/1658774184c4498991002cbb55f1714f 2024-11-15T04:41:31,265 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/1658774184c4498991002cbb55f1714f, entries=150, sequenceid=250, filesize=30.4 K 2024-11-15T04:41:31,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/3a88d42276cc4af1b86c3b2ca24d8058 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/3a88d42276cc4af1b86c3b2ca24d8058 2024-11-15T04:41:31,268 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/3a88d42276cc4af1b86c3b2ca24d8058, entries=150, sequenceid=250, filesize=11.9 K 2024-11-15T04:41:31,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/a5a552f5dae1481482563fcabda1e152 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/a5a552f5dae1481482563fcabda1e152 2024-11-15T04:41:31,271 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/a5a552f5dae1481482563fcabda1e152, entries=150, sequenceid=250, filesize=11.9 K 2024-11-15T04:41:31,272 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 05250181cbe972c6b3047f25758a3996 in 1293ms, sequenceid=250, compaction requested=false 2024-11-15T04:41:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-15T04:41:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-15T04:41:31,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-15T04:41:31,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3680 sec 2024-11-15T04:41:31,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.3720 sec 2024-11-15T04:41:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:31,386 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:31,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T04:41:31,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115939eaa7349fc4037a1a6d53e8c1a3114_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645691385/Put/seqid=0 2024-11-15T04:41:31,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742261_1437 (size=14994) 2024-11-15T04:41:31,400 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:31,404 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115939eaa7349fc4037a1a6d53e8c1a3114_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115939eaa7349fc4037a1a6d53e8c1a3114_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:31,405 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/67cdc3662c58431db49783f39565ce84, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:31,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/67cdc3662c58431db49783f39565ce84 is 175, key is test_row_0/A:col10/1731645691385/Put/seqid=0 2024-11-15T04:41:31,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742262_1438 (size=39949) 2024-11-15T04:41:31,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645751429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645751435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645751436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645751436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645751438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645751543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645751543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645751552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645751552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645751552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645751754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645751762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645751762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645751762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645751763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:31,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:31,813 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/67cdc3662c58431db49783f39565ce84 2024-11-15T04:41:31,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/f525f8ae474344bdbc0683f0af47c7d1 is 50, key is test_row_0/B:col10/1731645691385/Put/seqid=0 2024-11-15T04:41:31,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742263_1439 (size=12301) 2024-11-15T04:41:32,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645752063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:32,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645752073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:32,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645752073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:32,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645752073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:32,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645752082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:32,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/f525f8ae474344bdbc0683f0af47c7d1 2024-11-15T04:41:32,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/b549981043cc4bfa8ee9775dc0b01b56 is 50, key is test_row_0/C:col10/1731645691385/Put/seqid=0 2024-11-15T04:41:32,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742264_1440 (size=12301) 2024-11-15T04:41:32,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51638 deadline: 1731645752583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:32,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51592 deadline: 1731645752593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51594 deadline: 1731645752593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51650 deadline: 1731645752593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:32,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51652 deadline: 1731645752603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:41:32,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/b549981043cc4bfa8ee9775dc0b01b56 2024-11-15T04:41:32,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/67cdc3662c58431db49783f39565ce84 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/67cdc3662c58431db49783f39565ce84 2024-11-15T04:41:32,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/67cdc3662c58431db49783f39565ce84, entries=200, sequenceid=274, filesize=39.0 K 2024-11-15T04:41:32,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/f525f8ae474344bdbc0683f0af47c7d1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f525f8ae474344bdbc0683f0af47c7d1 2024-11-15T04:41:32,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f525f8ae474344bdbc0683f0af47c7d1, entries=150, sequenceid=274, filesize=12.0 K 2024-11-15T04:41:32,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/b549981043cc4bfa8ee9775dc0b01b56 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/b549981043cc4bfa8ee9775dc0b01b56 2024-11-15T04:41:32,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/b549981043cc4bfa8ee9775dc0b01b56, entries=150, sequenceid=274, filesize=12.0 K 2024-11-15T04:41:32,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 05250181cbe972c6b3047f25758a3996 in 1310ms, sequenceid=274, compaction requested=true 2024-11-15T04:41:32,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:32,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:32,696 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:32,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:32,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:32,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:32,696 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:32,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 05250181cbe972c6b3047f25758a3996:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:32,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:32,697 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:32,697 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:32,697 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/B is initiating minor compaction (all files) 2024-11-15T04:41:32,697 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/A is initiating minor compaction (all files) 2024-11-15T04:41:32,697 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/B in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:32,697 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/A in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:32,697 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/aa726dd3643240269de60459eb44e1be, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/1658774184c4498991002cbb55f1714f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/67cdc3662c58431db49783f39565ce84] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=100.3 K 2024-11-15T04:41:32,697 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/4044d057e12b456d8c0766aac6058a90, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/3a88d42276cc4af1b86c3b2ca24d8058, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f525f8ae474344bdbc0683f0af47c7d1] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=36.3 K 2024-11-15T04:41:32,697 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:32,697 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/aa726dd3643240269de60459eb44e1be, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/1658774184c4498991002cbb55f1714f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/67cdc3662c58431db49783f39565ce84] 2024-11-15T04:41:32,697 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 4044d057e12b456d8c0766aac6058a90, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731645688664 2024-11-15T04:41:32,697 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa726dd3643240269de60459eb44e1be, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731645688664 2024-11-15T04:41:32,698 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a88d42276cc4af1b86c3b2ca24d8058, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731645689028 2024-11-15T04:41:32,698 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1658774184c4498991002cbb55f1714f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731645689028 2024-11-15T04:41:32,698 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting f525f8ae474344bdbc0683f0af47c7d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731645690237 2024-11-15T04:41:32,698 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67cdc3662c58431db49783f39565ce84, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731645690237 2024-11-15T04:41:32,704 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:32,705 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411157f26ff72ea2343f6851bb2e76fa2abbf_05250181cbe972c6b3047f25758a3996 store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:32,706 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#B#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:32,706 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/487d961b67674fd49f03a126b6cb5a94 is 50, key is test_row_0/B:col10/1731645691385/Put/seqid=0 2024-11-15T04:41:32,708 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411157f26ff72ea2343f6851bb2e76fa2abbf_05250181cbe972c6b3047f25758a3996, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:32,708 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157f26ff72ea2343f6851bb2e76fa2abbf_05250181cbe972c6b3047f25758a3996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:32,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742265_1441 (size=12949) 2024-11-15T04:41:32,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742266_1442 (size=4469) 2024-11-15T04:41:32,991 INFO [Thread-1662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1662. 2024-11-15T04:41:32,991 DEBUG [Thread-1662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:32,991 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:32,991 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:32,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:32,994 INFO [Thread-1660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1660. 2024-11-15T04:41:32,994 DEBUG [Thread-1660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:32,994 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:32,994 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:32,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:32,994 INFO [Thread-1654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1654. 2024-11-15T04:41:32,994 DEBUG [Thread-1654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:32,994 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:32,994 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:32,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:32,998 INFO [Thread-1656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1656. 2024-11-15T04:41:32,998 DEBUG [Thread-1656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:32,998 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:32,998 DEBUG [Thread-1656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:32,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:33,000 INFO [Thread-1658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1658. 2024-11-15T04:41:33,000 DEBUG [Thread-1658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:33,000 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,000 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T04:41:33,073 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:33,114 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#A#compaction#368 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:33,115 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/12bb4e90543c422896a1691eb4bb2633 is 175, key is test_row_0/A:col10/1731645691385/Put/seqid=0 2024-11-15T04:41:33,119 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/487d961b67674fd49f03a126b6cb5a94 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/487d961b67674fd49f03a126b6cb5a94 2024-11-15T04:41:33,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742267_1443 (size=31903) 2024-11-15T04:41:33,125 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/B of 05250181cbe972c6b3047f25758a3996 into 487d961b67674fd49f03a126b6cb5a94(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:33,125 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:33,125 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/B, priority=13, startTime=1731645692696; duration=0sec 2024-11-15T04:41:33,125 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:33,125 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:B 2024-11-15T04:41:33,125 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:33,126 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:33,126 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 05250181cbe972c6b3047f25758a3996/C is initiating minor compaction (all files) 2024-11-15T04:41:33,126 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 05250181cbe972c6b3047f25758a3996/C in TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:33,126 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/58d30403ad044893ac97b104a5277a31, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/a5a552f5dae1481482563fcabda1e152, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/b549981043cc4bfa8ee9775dc0b01b56] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp, totalSize=36.3 K 2024-11-15T04:41:33,126 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 58d30403ad044893ac97b104a5277a31, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731645688664 2024-11-15T04:41:33,127 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting a5a552f5dae1481482563fcabda1e152, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731645689028 2024-11-15T04:41:33,127 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting b549981043cc4bfa8ee9775dc0b01b56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731645690237 2024-11-15T04:41:33,132 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 05250181cbe972c6b3047f25758a3996#C#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:33,133 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/e1431fc23add4484b8562b388c576a04 is 50, key is test_row_0/C:col10/1731645691385/Put/seqid=0 2024-11-15T04:41:33,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742268_1444 (size=12949) 2024-11-15T04:41:33,530 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/12bb4e90543c422896a1691eb4bb2633 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/12bb4e90543c422896a1691eb4bb2633 2024-11-15T04:41:33,537 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/A of 05250181cbe972c6b3047f25758a3996 into 12bb4e90543c422896a1691eb4bb2633(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:33,537 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:33,537 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/A, priority=13, startTime=1731645692696; duration=0sec 2024-11-15T04:41:33,537 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:33,537 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:A 2024-11-15T04:41:33,541 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/e1431fc23add4484b8562b388c576a04 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/e1431fc23add4484b8562b388c576a04 2024-11-15T04:41:33,546 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 05250181cbe972c6b3047f25758a3996/C of 05250181cbe972c6b3047f25758a3996 into e1431fc23add4484b8562b388c576a04(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:33,546 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:33,546 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., storeName=05250181cbe972c6b3047f25758a3996/C, priority=13, startTime=1731645692696; duration=0sec 2024-11-15T04:41:33,546 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:33,546 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 05250181cbe972c6b3047f25758a3996:C 2024-11-15T04:41:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:33,607 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:33,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T04:41:33,607 INFO [Thread-1649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1649. 2024-11-15T04:41:33,607 INFO [Thread-1645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1645. 2024-11-15T04:41:33,607 DEBUG [Thread-1649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:33,607 DEBUG [Thread-1645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:33,607 DEBUG [Thread-1649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,607 DEBUG [Thread-1645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,608 DEBUG [Thread-1645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,608 DEBUG [Thread-1649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:33,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:33,615 INFO [Thread-1651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1651. 2024-11-15T04:41:33,615 INFO [Thread-1643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1643. 2024-11-15T04:41:33,615 DEBUG [Thread-1651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:33,615 DEBUG [Thread-1643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:33,615 DEBUG [Thread-1651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,615 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,615 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,615 DEBUG [Thread-1651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:33,615 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:33,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115938c656c75004b67a0d2f49d3c24baa8_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645691435/Put/seqid=0 2024-11-15T04:41:33,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742269_1445 (size=12454) 2024-11-15T04:41:33,624 INFO [Thread-1647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1647. 2024-11-15T04:41:33,624 DEBUG [Thread-1647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:41:33,624 DEBUG [Thread-1647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,624 DEBUG [Thread-1647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:33,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T04:41:33,624 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1258 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3771 rows 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1259 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3771 rows 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1256 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3765 rows 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1255 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3762 rows 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1247 2024-11-15T04:41:33,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3741 rows 2024-11-15T04:41:33,625 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T04:41:33,625 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T04:41:33,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:33,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:41:33,626 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T04:41:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T04:41:33,628 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645693628"}]},"ts":"1731645693628"} 2024-11-15T04:41:33,629 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T04:41:33,630 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T04:41:33,630 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T04:41:33,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, UNASSIGN}] 2024-11-15T04:41:33,633 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, UNASSIGN 2024-11-15T04:41:33,634 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=05250181cbe972c6b3047f25758a3996, regionState=CLOSING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:33,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, UNASSIGN because future has completed 2024-11-15T04:41:33,635 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T04:41:33,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:41:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T04:41:33,788 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:33,788 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T04:41:33,788 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 05250181cbe972c6b3047f25758a3996, disabling compactions & flushes 2024-11-15T04:41:33,788 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T04:41:34,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:34,025 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115938c656c75004b67a0d2f49d3c24baa8_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115938c656c75004b67a0d2f49d3c24baa8_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:34,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/7c621b1c970a4292bd9b19482a0c53f5, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:34,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/7c621b1c970a4292bd9b19482a0c53f5 is 175, key is test_row_0/A:col10/1731645691435/Put/seqid=0 2024-11-15T04:41:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742270_1446 (size=31255) 2024-11-15T04:41:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T04:41:34,430 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/7c621b1c970a4292bd9b19482a0c53f5 2024-11-15T04:41:34,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/8faa9d234cb1487d821ebced99eb326f is 50, key is test_row_0/B:col10/1731645691435/Put/seqid=0 2024-11-15T04:41:34,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742271_1447 (size=12301) 2024-11-15T04:41:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T04:41:34,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/8faa9d234cb1487d821ebced99eb326f 2024-11-15T04:41:34,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/56d31adc304c474b94fff245b233d757 is 50, key is test_row_0/C:col10/1731645691435/Put/seqid=0 2024-11-15T04:41:34,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742272_1448 (size=12301) 2024-11-15T04:41:35,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/56d31adc304c474b94fff245b233d757 2024-11-15T04:41:35,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/7c621b1c970a4292bd9b19482a0c53f5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/7c621b1c970a4292bd9b19482a0c53f5 2024-11-15T04:41:35,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/7c621b1c970a4292bd9b19482a0c53f5, entries=150, sequenceid=294, filesize=30.5 K 2024-11-15T04:41:35,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/8faa9d234cb1487d821ebced99eb326f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/8faa9d234cb1487d821ebced99eb326f 2024-11-15T04:41:35,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/8faa9d234cb1487d821ebced99eb326f, entries=150, sequenceid=294, filesize=12.0 K 2024-11-15T04:41:35,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/56d31adc304c474b94fff245b233d757 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/56d31adc304c474b94fff245b233d757 2024-11-15T04:41:35,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/56d31adc304c474b94fff245b233d757, entries=150, sequenceid=294, filesize=12.0 K 2024-11-15T04:41:35,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=20.13 KB/20610 for 05250181cbe972c6b3047f25758a3996 in 1672ms, sequenceid=294, compaction requested=false 2024-11-15T04:41:35,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 05250181cbe972c6b3047f25758a3996: 2024-11-15T04:41:35,279 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:35,279 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:35,279 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. after waiting 0 ms 2024-11-15T04:41:35,279 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:35,279 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 05250181cbe972c6b3047f25758a3996 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-15T04:41:35,283 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157246fe15ebd94ee8a3881ddd6c49a252_05250181cbe972c6b3047f25758a3996 is 50, key is test_row_0/A:col10/1731645693623/Put/seqid=0 2024-11-15T04:41:35,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742273_1449 (size=7374) 2024-11-15T04:41:35,688 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:35,694 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157246fe15ebd94ee8a3881ddd6c49a252_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157246fe15ebd94ee8a3881ddd6c49a252_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:35,695 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/0028510fd8ef4b588844133a482594d3, store: [table=TestAcidGuarantees family=A region=05250181cbe972c6b3047f25758a3996] 2024-11-15T04:41:35,696 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/0028510fd8ef4b588844133a482594d3 is 175, key is test_row_0/A:col10/1731645693623/Put/seqid=0 2024-11-15T04:41:35,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742274_1450 (size=13865) 2024-11-15T04:41:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T04:41:36,100 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/0028510fd8ef4b588844133a482594d3 2024-11-15T04:41:36,110 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/1e6c7086ebdd4d95b12e77ff9503924d is 50, key is test_row_0/B:col10/1731645693623/Put/seqid=0 2024-11-15T04:41:36,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742275_1451 (size=7415) 2024-11-15T04:41:36,514 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/1e6c7086ebdd4d95b12e77ff9503924d 2024-11-15T04:41:36,525 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/fa87113074764e469bf0d706c7c1f145 is 50, key is test_row_0/C:col10/1731645693623/Put/seqid=0 2024-11-15T04:41:36,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742276_1452 (size=7415) 2024-11-15T04:41:36,931 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/fa87113074764e469bf0d706c7c1f145 2024-11-15T04:41:36,973 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/A/0028510fd8ef4b588844133a482594d3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/0028510fd8ef4b588844133a482594d3 2024-11-15T04:41:36,979 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/0028510fd8ef4b588844133a482594d3, entries=50, sequenceid=300, filesize=13.5 K 2024-11-15T04:41:36,980 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/B/1e6c7086ebdd4d95b12e77ff9503924d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/1e6c7086ebdd4d95b12e77ff9503924d 2024-11-15T04:41:36,984 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/1e6c7086ebdd4d95b12e77ff9503924d, entries=50, sequenceid=300, filesize=7.2 K 2024-11-15T04:41:36,985 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/.tmp/C/fa87113074764e469bf0d706c7c1f145 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/fa87113074764e469bf0d706c7c1f145 2024-11-15T04:41:36,988 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/fa87113074764e469bf0d706c7c1f145, entries=50, sequenceid=300, filesize=7.2 K 2024-11-15T04:41:36,989 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 05250181cbe972c6b3047f25758a3996 in 1710ms, sequenceid=300, compaction requested=true 2024-11-15T04:41:36,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/9af6cca0a9f8432f84c208f81cbb14bd, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/6e098871a1c64664b2e0e2128ed4274d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8cfbb1b1877744788be2a4f040321ffd, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/60f05b03dfeb4f58999ddf3e4272e23c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d834aad3ccce48a8ae69408154411663, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/35e6444e94544dfcbb94c0ff825cb9d3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d5da0077d86a4946a80c8e4289a27d9a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d40ae2a9cb82402c802d7691bca29225, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/0d7c10ef15014a0fbfadf0264c0dcd9f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/f204fd5d66ab4db99187a2832deacadd, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/a311ad2e9a494a14b35da2f3fcd91213, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/db5652c09ac941fcb8d26961a13d7d29, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/39468c9d75be47ccb86297575218f61f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/54f9e805cd5a43cabaa50eadad6d49ab, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/81ac4e0f7ec24730b9f114193861d051, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8dc0de908a2642ffa8866b47a10441c8, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/aa726dd3643240269de60459eb44e1be, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/1658774184c4498991002cbb55f1714f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/67cdc3662c58431db49783f39565ce84] to archive 2024-11-15T04:41:36,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:41:36,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/9af6cca0a9f8432f84c208f81cbb14bd to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/9af6cca0a9f8432f84c208f81cbb14bd 2024-11-15T04:41:36,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/6e098871a1c64664b2e0e2128ed4274d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/6e098871a1c64664b2e0e2128ed4274d 2024-11-15T04:41:36,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8cfbb1b1877744788be2a4f040321ffd to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8cfbb1b1877744788be2a4f040321ffd 2024-11-15T04:41:36,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/60f05b03dfeb4f58999ddf3e4272e23c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/60f05b03dfeb4f58999ddf3e4272e23c 2024-11-15T04:41:36,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d834aad3ccce48a8ae69408154411663 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d834aad3ccce48a8ae69408154411663 2024-11-15T04:41:36,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/35e6444e94544dfcbb94c0ff825cb9d3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/35e6444e94544dfcbb94c0ff825cb9d3 2024-11-15T04:41:36,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d5da0077d86a4946a80c8e4289a27d9a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d5da0077d86a4946a80c8e4289a27d9a 2024-11-15T04:41:36,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d40ae2a9cb82402c802d7691bca29225 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/d40ae2a9cb82402c802d7691bca29225 2024-11-15T04:41:36,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/0d7c10ef15014a0fbfadf0264c0dcd9f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/0d7c10ef15014a0fbfadf0264c0dcd9f 2024-11-15T04:41:36,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/f204fd5d66ab4db99187a2832deacadd to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/f204fd5d66ab4db99187a2832deacadd 2024-11-15T04:41:37,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/a311ad2e9a494a14b35da2f3fcd91213 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/a311ad2e9a494a14b35da2f3fcd91213 2024-11-15T04:41:37,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/db5652c09ac941fcb8d26961a13d7d29 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/db5652c09ac941fcb8d26961a13d7d29 2024-11-15T04:41:37,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/39468c9d75be47ccb86297575218f61f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/39468c9d75be47ccb86297575218f61f 2024-11-15T04:41:37,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/54f9e805cd5a43cabaa50eadad6d49ab to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/54f9e805cd5a43cabaa50eadad6d49ab 2024-11-15T04:41:37,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/81ac4e0f7ec24730b9f114193861d051 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/81ac4e0f7ec24730b9f114193861d051 2024-11-15T04:41:37,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8dc0de908a2642ffa8866b47a10441c8 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/8dc0de908a2642ffa8866b47a10441c8 2024-11-15T04:41:37,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/aa726dd3643240269de60459eb44e1be to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/aa726dd3643240269de60459eb44e1be 2024-11-15T04:41:37,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/1658774184c4498991002cbb55f1714f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/1658774184c4498991002cbb55f1714f 2024-11-15T04:41:37,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/67cdc3662c58431db49783f39565ce84 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/67cdc3662c58431db49783f39565ce84 2024-11-15T04:41:37,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/2445c0237f0b40ebb7b321de0b40975c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/fc78ed97ddd642c2b67b7aa8fdce7e72, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/4d218eb505334a03811b313cd379d359, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f821fe44fe334e6c8a84fd929cf2171e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/8ceed50673bc467c9b5bec053b0172fb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/198a28a8d50b46679a0aafcd6b03e9d3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/26f28eb3dec24e0eb502b44647833bd3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/171dd355621b461b9aec68f8286f2200, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/9ac76a17c0c44eeda8a43eaccb03c661, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/897f081806654b349576f858edb40609, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/df5d427906504ef9a464578cdc36addd, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/bdade7e0d0ab4f2899607c567fb6bdee, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/15a5e2e4c2844f699edb4d4c00b8aeca, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/6dbc7de3f2f9429aa000644e91c4c763, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/7f8906879c514c9e8f7471b5ac650014, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/4044d057e12b456d8c0766aac6058a90, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/db9da9b0231f49eebed955f86c2184cb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/3a88d42276cc4af1b86c3b2ca24d8058, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f525f8ae474344bdbc0683f0af47c7d1] to archive 2024-11-15T04:41:37,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:41:37,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/2445c0237f0b40ebb7b321de0b40975c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/2445c0237f0b40ebb7b321de0b40975c 2024-11-15T04:41:37,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/fc78ed97ddd642c2b67b7aa8fdce7e72 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/fc78ed97ddd642c2b67b7aa8fdce7e72 2024-11-15T04:41:37,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/4d218eb505334a03811b313cd379d359 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/4d218eb505334a03811b313cd379d359 2024-11-15T04:41:37,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f821fe44fe334e6c8a84fd929cf2171e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f821fe44fe334e6c8a84fd929cf2171e 2024-11-15T04:41:37,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/8ceed50673bc467c9b5bec053b0172fb to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/8ceed50673bc467c9b5bec053b0172fb 2024-11-15T04:41:37,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/198a28a8d50b46679a0aafcd6b03e9d3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/198a28a8d50b46679a0aafcd6b03e9d3 2024-11-15T04:41:37,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/26f28eb3dec24e0eb502b44647833bd3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/26f28eb3dec24e0eb502b44647833bd3 2024-11-15T04:41:37,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/171dd355621b461b9aec68f8286f2200 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/171dd355621b461b9aec68f8286f2200 2024-11-15T04:41:37,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/9ac76a17c0c44eeda8a43eaccb03c661 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/9ac76a17c0c44eeda8a43eaccb03c661 2024-11-15T04:41:37,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/897f081806654b349576f858edb40609 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/897f081806654b349576f858edb40609 2024-11-15T04:41:37,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/df5d427906504ef9a464578cdc36addd to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/df5d427906504ef9a464578cdc36addd 2024-11-15T04:41:37,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/bdade7e0d0ab4f2899607c567fb6bdee to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/bdade7e0d0ab4f2899607c567fb6bdee 2024-11-15T04:41:37,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/15a5e2e4c2844f699edb4d4c00b8aeca to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/15a5e2e4c2844f699edb4d4c00b8aeca 2024-11-15T04:41:37,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/6dbc7de3f2f9429aa000644e91c4c763 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/6dbc7de3f2f9429aa000644e91c4c763 2024-11-15T04:41:37,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/7f8906879c514c9e8f7471b5ac650014 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/7f8906879c514c9e8f7471b5ac650014 2024-11-15T04:41:37,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/4044d057e12b456d8c0766aac6058a90 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/4044d057e12b456d8c0766aac6058a90 2024-11-15T04:41:37,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/db9da9b0231f49eebed955f86c2184cb to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/db9da9b0231f49eebed955f86c2184cb 2024-11-15T04:41:37,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/3a88d42276cc4af1b86c3b2ca24d8058 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/3a88d42276cc4af1b86c3b2ca24d8058 2024-11-15T04:41:37,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f525f8ae474344bdbc0683f0af47c7d1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/f525f8ae474344bdbc0683f0af47c7d1 2024-11-15T04:41:37,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/d2febb30c8c842e28025f1cc9a36a8d1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/c65d99bbf3274a9e922bc94cfb7d6083, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/1eb1666b7c4d4b5fba5872d530baac3d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/2373960e75dc4143841d101fd4acce84, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/e356541ac0b040bd998690f6a78fd47f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/fee98083d8684a23b7f667c739a22b7c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/23b2f949cdd34c4abeab7a16542b50f9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/70379ebc4dd9440a9a8d2682b659ca54, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/26371d7a9b3c4edf8f53a0e9ba546c59, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/05165d6af54d4507b1af21895d008859, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/38c2e0020b4d47b8befdedb4044fdd5f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/66e3eeb6f4604f379bae8db991af06e7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/9c6c2f21cb0c48388e8b3eeaf32d6276, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/f8da1688c3ea44ae8cf2f6fd4e3c71df, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/3964387923b6432aa6ee88747f55a9a6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/58d30403ad044893ac97b104a5277a31, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/ef83ed4325d4461fb3c61055457dd3c1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/a5a552f5dae1481482563fcabda1e152, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/b549981043cc4bfa8ee9775dc0b01b56] to archive 2024-11-15T04:41:37,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:41:37,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/d2febb30c8c842e28025f1cc9a36a8d1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/d2febb30c8c842e28025f1cc9a36a8d1 2024-11-15T04:41:37,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/c65d99bbf3274a9e922bc94cfb7d6083 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/c65d99bbf3274a9e922bc94cfb7d6083 2024-11-15T04:41:37,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/1eb1666b7c4d4b5fba5872d530baac3d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/1eb1666b7c4d4b5fba5872d530baac3d 2024-11-15T04:41:37,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/2373960e75dc4143841d101fd4acce84 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/2373960e75dc4143841d101fd4acce84 2024-11-15T04:41:37,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/e356541ac0b040bd998690f6a78fd47f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/e356541ac0b040bd998690f6a78fd47f 2024-11-15T04:41:37,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/fee98083d8684a23b7f667c739a22b7c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/fee98083d8684a23b7f667c739a22b7c 2024-11-15T04:41:37,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/23b2f949cdd34c4abeab7a16542b50f9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/23b2f949cdd34c4abeab7a16542b50f9 2024-11-15T04:41:37,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/70379ebc4dd9440a9a8d2682b659ca54 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/70379ebc4dd9440a9a8d2682b659ca54 2024-11-15T04:41:37,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/26371d7a9b3c4edf8f53a0e9ba546c59 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/26371d7a9b3c4edf8f53a0e9ba546c59 2024-11-15T04:41:37,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/05165d6af54d4507b1af21895d008859 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/05165d6af54d4507b1af21895d008859 2024-11-15T04:41:37,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/38c2e0020b4d47b8befdedb4044fdd5f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/38c2e0020b4d47b8befdedb4044fdd5f 2024-11-15T04:41:37,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/66e3eeb6f4604f379bae8db991af06e7 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/66e3eeb6f4604f379bae8db991af06e7 2024-11-15T04:41:37,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/9c6c2f21cb0c48388e8b3eeaf32d6276 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/9c6c2f21cb0c48388e8b3eeaf32d6276 2024-11-15T04:41:37,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/f8da1688c3ea44ae8cf2f6fd4e3c71df to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/f8da1688c3ea44ae8cf2f6fd4e3c71df 2024-11-15T04:41:37,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/3964387923b6432aa6ee88747f55a9a6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/3964387923b6432aa6ee88747f55a9a6 2024-11-15T04:41:37,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/58d30403ad044893ac97b104a5277a31 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/58d30403ad044893ac97b104a5277a31 2024-11-15T04:41:37,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/ef83ed4325d4461fb3c61055457dd3c1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/ef83ed4325d4461fb3c61055457dd3c1 2024-11-15T04:41:37,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/a5a552f5dae1481482563fcabda1e152 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/a5a552f5dae1481482563fcabda1e152 2024-11-15T04:41:37,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/b549981043cc4bfa8ee9775dc0b01b56 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/b549981043cc4bfa8ee9775dc0b01b56 2024-11-15T04:41:37,038 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/recovered.edits/303.seqid, newMaxSeqId=303, maxSeqId=4 2024-11-15T04:41:37,039 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. 2024-11-15T04:41:37,039 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 05250181cbe972c6b3047f25758a3996: Waiting for close lock at 1731645693788Running coprocessor pre-close hooks at 1731645693788Disabling compacts and flushes for region at 1731645693788Disabling writes for close at 1731645695279 (+1491 ms)Obtaining lock to block concurrent updates at 1731645695279Preparing flush snapshotting stores in 05250181cbe972c6b3047f25758a3996 at 1731645695279Finished memstore snapshotting TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731645695279Flushing stores of TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996. at 1731645695280 (+1 ms)Flushing 05250181cbe972c6b3047f25758a3996/A: creating writer at 1731645695280Flushing mob file 05250181cbe972c6b3047f25758a3996/A: appending metadata at 1731645695283 (+3 ms)Flushing mob file 05250181cbe972c6b3047f25758a3996/A: closing flushed file at 1731645695283Flushing 05250181cbe972c6b3047f25758a3996/A: appending metadata at 1731645695695 (+412 ms)Flushing 05250181cbe972c6b3047f25758a3996/A: closing flushed file at 1731645695695Flushing 05250181cbe972c6b3047f25758a3996/B: creating writer at 1731645696107 (+412 ms)Flushing 05250181cbe972c6b3047f25758a3996/B: appending metadata at 1731645696110 (+3 ms)Flushing 05250181cbe972c6b3047f25758a3996/B: closing flushed file at 1731645696110Flushing 05250181cbe972c6b3047f25758a3996/C: creating writer at 1731645696520 (+410 ms)Flushing 05250181cbe972c6b3047f25758a3996/C: appending metadata at 1731645696524 (+4 ms)Flushing 05250181cbe972c6b3047f25758a3996/C: closing flushed file at 1731645696524Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51d29879: reopening flushed file at 1731645696972 (+448 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fe0b68f: reopening flushed file at 1731645696979 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72a60aec: reopening flushed file at 1731645696984 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 05250181cbe972c6b3047f25758a3996 in 1710ms, sequenceid=300, compaction requested=true at 1731645696989 (+5 ms)Writing region close event to WAL at 1731645697036 (+47 ms)Running coprocessor post-close hooks at 1731645697039 (+3 ms)Closed at 1731645697039 2024-11-15T04:41:37,040 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,041 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=05250181cbe972c6b3047f25758a3996, regionState=CLOSED 2024-11-15T04:41:37,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:41:37,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-15T04:41:37,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 05250181cbe972c6b3047f25758a3996, server=26747e6ada4f,42935,1731645581086 in 3.4080 sec 2024-11-15T04:41:37,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-15T04:41:37,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=05250181cbe972c6b3047f25758a3996, UNASSIGN in 3.4120 sec 2024-11-15T04:41:37,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-15T04:41:37,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.4150 sec 2024-11-15T04:41:37,048 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645697048"}]},"ts":"1731645697048"} 2024-11-15T04:41:37,049 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T04:41:37,049 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T04:41:37,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.4240 sec 2024-11-15T04:41:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T04:41:37,793 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:37,794 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T04:41:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:37,798 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T04:41:37,800 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:37,805 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,809 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/recovered.edits] 2024-11-15T04:41:37,814 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/0028510fd8ef4b588844133a482594d3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/0028510fd8ef4b588844133a482594d3 2024-11-15T04:41:37,815 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/12bb4e90543c422896a1691eb4bb2633 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/12bb4e90543c422896a1691eb4bb2633 2024-11-15T04:41:37,817 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/7c621b1c970a4292bd9b19482a0c53f5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/A/7c621b1c970a4292bd9b19482a0c53f5 2024-11-15T04:41:37,820 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/1e6c7086ebdd4d95b12e77ff9503924d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/1e6c7086ebdd4d95b12e77ff9503924d 2024-11-15T04:41:37,822 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/487d961b67674fd49f03a126b6cb5a94 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/487d961b67674fd49f03a126b6cb5a94 2024-11-15T04:41:37,823 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/8faa9d234cb1487d821ebced99eb326f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/B/8faa9d234cb1487d821ebced99eb326f 2024-11-15T04:41:37,826 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/56d31adc304c474b94fff245b233d757 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/56d31adc304c474b94fff245b233d757 2024-11-15T04:41:37,828 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/e1431fc23add4484b8562b388c576a04 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/e1431fc23add4484b8562b388c576a04 2024-11-15T04:41:37,830 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/fa87113074764e469bf0d706c7c1f145 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/C/fa87113074764e469bf0d706c7c1f145 2024-11-15T04:41:37,833 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/recovered.edits/303.seqid to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996/recovered.edits/303.seqid 2024-11-15T04:41:37,834 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,834 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T04:41:37,834 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T04:41:37,835 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T04:41:37,838 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111506340d2a58894176838d5f0e02b32532_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111506340d2a58894176838d5f0e02b32532_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,840 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150648ca5b4f47445486ba66ee11e64f0a_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150648ca5b4f47445486ba66ee11e64f0a_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,841 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111519de19c0411343db97493af5097c3968_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111519de19c0411343db97493af5097c3968_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,843 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151a29a63b29594609873a7ed64f6b49e1_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151a29a63b29594609873a7ed64f6b49e1_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,844 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111536ffd46d74ca4eb99a6b60a9646ca0f6_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111536ffd46d74ca4eb99a6b60a9646ca0f6_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,846 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155b152c1b04aa4d8ca20745e76537069c_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155b152c1b04aa4d8ca20745e76537069c_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,847 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111563b16297dc2845afa1a468550e608fec_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111563b16297dc2845afa1a468550e608fec_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,849 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156da81b86e6bb4d7e9cfc9f2f3f6e7b7d_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156da81b86e6bb4d7e9cfc9f2f3f6e7b7d_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,850 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157246fe15ebd94ee8a3881ddd6c49a252_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157246fe15ebd94ee8a3881ddd6c49a252_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,852 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115938c656c75004b67a0d2f49d3c24baa8_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115938c656c75004b67a0d2f49d3c24baa8_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,853 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115939eaa7349fc4037a1a6d53e8c1a3114_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115939eaa7349fc4037a1a6d53e8c1a3114_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,854 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b3225c15722b42feacd914f2aaf25b5a_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b3225c15722b42feacd914f2aaf25b5a_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,856 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4755699c5ee47469c7dd2e8272dc692_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b4755699c5ee47469c7dd2e8272dc692_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,857 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c742d2d4bcd44eaeb617495324eec2c2_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c742d2d4bcd44eaeb617495324eec2c2_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,858 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fa088d7faeab4ea1822aa4b5bcea8db4_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115fa088d7faeab4ea1822aa4b5bcea8db4_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,860 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ff7e7a9ff86f4336b493d002677c6548_05250181cbe972c6b3047f25758a3996 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ff7e7a9ff86f4336b493d002677c6548_05250181cbe972c6b3047f25758a3996 2024-11-15T04:41:37,860 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T04:41:37,862 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:37,865 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T04:41:37,867 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T04:41:37,868 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:37,868 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T04:41:37,868 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731645697868"}]},"ts":"9223372036854775807"} 2024-11-15T04:41:37,870 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T04:41:37,870 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 05250181cbe972c6b3047f25758a3996, NAME => 'TestAcidGuarantees,,1731645669843.05250181cbe972c6b3047f25758a3996.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T04:41:37,870 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T04:41:37,871 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731645697870"}]},"ts":"9223372036854775807"} 2024-11-15T04:41:37,872 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T04:41:37,873 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:37,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-11-15T04:41:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-15T04:41:37,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T04:41:37,903 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:37,910 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 239) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 535), ProcessCount=11 (was 11), AvailableMemoryMB=14668 (was 14198) - AvailableMemoryMB LEAK? - 2024-11-15T04:41:37,917 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=14668 2024-11-15T04:41:37,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T04:41:37,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T04:41:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T04:41:37,920 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T04:41:37,920 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:37,920 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-15T04:41:37,921 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T04:41:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-15T04:41:37,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742277_1453 (size=959) 2024-11-15T04:41:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-15T04:41:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-15T04:41:38,327 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6 2024-11-15T04:41:38,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742278_1454 (size=53) 2024-11-15T04:41:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-15T04:41:38,733 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:41:38,733 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c079ffac046c5f34a718ffcef10b362b, disabling compactions & flushes 2024-11-15T04:41:38,734 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:38,734 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:38,734 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. after waiting 0 ms 2024-11-15T04:41:38,734 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:38,734 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:38,734 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c079ffac046c5f34a718ffcef10b362b: Waiting for close lock at 1731645698733Disabling compacts and flushes for region at 1731645698733Disabling writes for close at 1731645698734 (+1 ms)Writing region close event to WAL at 1731645698734Closed at 1731645698734 2024-11-15T04:41:38,736 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T04:41:38,737 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731645698736"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731645698736"}]},"ts":"1731645698736"} 2024-11-15T04:41:38,740 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T04:41:38,742 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T04:41:38,742 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645698742"}]},"ts":"1731645698742"} 2024-11-15T04:41:38,744 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T04:41:38,744 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c079ffac046c5f34a718ffcef10b362b, ASSIGN}] 2024-11-15T04:41:38,745 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c079ffac046c5f34a718ffcef10b362b, ASSIGN 2024-11-15T04:41:38,746 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c079ffac046c5f34a718ffcef10b362b, ASSIGN; state=OFFLINE, location=26747e6ada4f,42935,1731645581086; forceNewPlan=false, retain=false 2024-11-15T04:41:38,897 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=c079ffac046c5f34a718ffcef10b362b, regionState=OPENING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:38,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c079ffac046c5f34a718ffcef10b362b, ASSIGN because future has completed 2024-11-15T04:41:38,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-15T04:41:39,058 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:39,059 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:41:39,060 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,060 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:41:39,060 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,060 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,062 INFO [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,063 INFO [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c079ffac046c5f34a718ffcef10b362b columnFamilyName A 2024-11-15T04:41:39,063 DEBUG [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:39,064 INFO [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] regionserver.HStore(327): Store=c079ffac046c5f34a718ffcef10b362b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:41:39,064 INFO [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,066 INFO [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c079ffac046c5f34a718ffcef10b362b columnFamilyName B 2024-11-15T04:41:39,066 DEBUG [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:39,067 INFO [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] regionserver.HStore(327): Store=c079ffac046c5f34a718ffcef10b362b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:41:39,067 INFO [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,069 INFO [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c079ffac046c5f34a718ffcef10b362b columnFamilyName C 2024-11-15T04:41:39,069 DEBUG [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:41:39,070 INFO [StoreOpener-c079ffac046c5f34a718ffcef10b362b-1 {}] regionserver.HStore(327): Store=c079ffac046c5f34a718ffcef10b362b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:41:39,070 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,071 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,071 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,073 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,073 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,073 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:41:39,075 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,078 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T04:41:39,079 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened c079ffac046c5f34a718ffcef10b362b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67457125, jitterRate=0.005189493298530579}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:41:39,079 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:39,079 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for c079ffac046c5f34a718ffcef10b362b: Running coprocessor pre-open hook at 1731645699060Writing region info on filesystem at 1731645699060Initializing all the Stores at 1731645699061 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645699061Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645699061Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645699061Cleaning up temporary data from old regions at 1731645699073 (+12 ms)Running coprocessor post-open hooks at 1731645699079 (+6 ms)Region opened successfully at 1731645699079 2024-11-15T04:41:39,080 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., pid=116, masterSystemTime=1731645699055 2024-11-15T04:41:39,081 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:39,081 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:39,082 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=c079ffac046c5f34a718ffcef10b362b, regionState=OPEN, openSeqNum=2, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:39,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:41:39,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-15T04:41:39,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 in 183 msec 2024-11-15T04:41:39,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-15T04:41:39,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c079ffac046c5f34a718ffcef10b362b, ASSIGN in 341 msec 2024-11-15T04:41:39,088 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T04:41:39,088 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645699088"}]},"ts":"1731645699088"} 2024-11-15T04:41:39,089 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T04:41:39,089 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T04:41:39,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-11-15T04:41:39,361 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T04:41:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-15T04:41:40,064 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:40,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa9e45f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,066 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,067 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aec684a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,070 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72809a3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,071 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,075 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T04:41:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 26747e6ada4f,42655,1731645580243 2024-11-15T04:41:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@9ab1811 2024-11-15T04:41:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T04:41:40,078 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T04:41:40,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3734923a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,080 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@671fd9c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,081 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6010d228, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,082 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f704f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,082 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,083 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663f27b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,084 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b24517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b7894c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,086 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b9887c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,087 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e79d367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1447b9d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,088 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,088 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,089 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9c84564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,090 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b56af75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1de6e594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,091 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,091 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,092 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@473513c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,093 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e1b4e71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f86346d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,095 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,095 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@94c6b62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,097 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45089281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b469189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,099 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109dbbb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,100 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50e6a733, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6240d801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,102 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11707356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,103 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5dbde3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e2d9ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,104 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24e05388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,105 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1315bb55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d9f546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:41:40,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:41:40,107 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:41:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:41:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:41:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@84bd82e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:41:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:41:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:41:40,108 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:41:40,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e0cf3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:41:40,109 DEBUG [Thread-2047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,109 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,109 DEBUG [Thread-2041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,109 DEBUG [Thread-2043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,110 DEBUG [Thread-2045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,111 DEBUG [Thread-2034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-15T04:41:40,112 DEBUG [Thread-2049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,112 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,113 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T04:41:40,113 DEBUG [Thread-2036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,113 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,115 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:40,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:41:40,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:41:40,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:41:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:41:40,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:41:40,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:41:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:41:40,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:41:40,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:41:40,121 DEBUG [Thread-2038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:41:40,122 DEBUG [Thread-2038 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:41:40,123 DEBUG [Thread-2038 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:41:40,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:41:40,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2] 2024-11-15T04:41:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:40,129 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:40,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:41:40,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/981d7de07cd145cebeaf188b05a4d9a1 is 50, key is test_row_0/A:col10/1731645700129/Put/seqid=0 2024-11-15T04:41:40,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742279_1455 (size=12001) 2024-11-15T04:41:40,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/981d7de07cd145cebeaf188b05a4d9a1 2024-11-15T04:41:40,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645760161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645760162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645760163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645760164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645760169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/5e43d962cc9a4c09a5c48becf4fca97f is 50, key is test_row_0/B:col10/1731645700129/Put/seqid=0 2024-11-15T04:41:40,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742280_1456 (size=12001) 2024-11-15T04:41:40,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T04:41:40,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T04:41:40,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:40,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:40,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:40,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:40,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:40,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:40,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645760272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645760272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645760283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645760283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645760283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T04:41:40,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:40,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:40,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:40,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:40,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T04:41:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645760482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645760483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645760493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645760493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645760493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T04:41:40,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:40,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:40,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:40,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:40,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/5e43d962cc9a4c09a5c48becf4fca97f 2024-11-15T04:41:40,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/0fe657d52a834fd4964b983cd07c75e8 is 50, key is test_row_0/C:col10/1731645700129/Put/seqid=0 2024-11-15T04:41:40,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742281_1457 (size=12001) 2024-11-15T04:41:40,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/0fe657d52a834fd4964b983cd07c75e8 2024-11-15T04:41:40,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/981d7de07cd145cebeaf188b05a4d9a1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/981d7de07cd145cebeaf188b05a4d9a1 2024-11-15T04:41:40,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/981d7de07cd145cebeaf188b05a4d9a1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T04:41:40,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/5e43d962cc9a4c09a5c48becf4fca97f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/5e43d962cc9a4c09a5c48becf4fca97f 2024-11-15T04:41:40,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/5e43d962cc9a4c09a5c48becf4fca97f, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T04:41:40,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/0fe657d52a834fd4964b983cd07c75e8 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/0fe657d52a834fd4964b983cd07c75e8 2024-11-15T04:41:40,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/0fe657d52a834fd4964b983cd07c75e8, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T04:41:40,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c079ffac046c5f34a718ffcef10b362b in 514ms, sequenceid=14, compaction requested=false 2024-11-15T04:41:40,643 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T04:41:40,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:40,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-15T04:41:40,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:40,726 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T04:41:40,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/83a1a6e92bdb481584a35dacc778b688 is 50, key is test_row_0/A:col10/1731645700162/Put/seqid=0 2024-11-15T04:41:40,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742282_1458 (size=12001) 2024-11-15T04:41:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T04:41:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:40,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:40,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645760807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645760811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645760812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645760813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645760813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645760922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645760922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645760922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645760933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645760933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,133 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/83a1a6e92bdb481584a35dacc778b688 2024-11-15T04:41:41,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645761132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645761132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645761132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/39875920814b49c5abc259fa7b3bb6df is 50, key is test_row_0/B:col10/1731645700162/Put/seqid=0 2024-11-15T04:41:41,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742283_1459 (size=12001) 2024-11-15T04:41:41,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645761143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645761143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T04:41:41,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645761442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645761442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645761442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645761452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645761453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,545 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/39875920814b49c5abc259fa7b3bb6df 2024-11-15T04:41:41,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/bdb6fb122ba04e049474683431d1a2e4 is 50, key is test_row_0/C:col10/1731645700162/Put/seqid=0 2024-11-15T04:41:41,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742284_1460 (size=12001) 2024-11-15T04:41:41,955 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/bdb6fb122ba04e049474683431d1a2e4 2024-11-15T04:41:41,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645761952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645761963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645761963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645761963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645761963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:41,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:42,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/83a1a6e92bdb481584a35dacc778b688 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/83a1a6e92bdb481584a35dacc778b688 2024-11-15T04:41:42,004 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/83a1a6e92bdb481584a35dacc778b688, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T04:41:42,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/39875920814b49c5abc259fa7b3bb6df as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/39875920814b49c5abc259fa7b3bb6df 2024-11-15T04:41:42,008 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/39875920814b49c5abc259fa7b3bb6df, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T04:41:42,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/bdb6fb122ba04e049474683431d1a2e4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/bdb6fb122ba04e049474683431d1a2e4 2024-11-15T04:41:42,012 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/bdb6fb122ba04e049474683431d1a2e4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T04:41:42,012 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c079ffac046c5f34a718ffcef10b362b in 1286ms, sequenceid=37, compaction requested=false 2024-11-15T04:41:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-15T04:41:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-15T04:41:42,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-15T04:41:42,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8980 sec 2024-11-15T04:41:42,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.9060 sec 2024-11-15T04:41:42,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-15T04:41:42,262 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:42,263 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-15T04:41:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T04:41:42,264 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:42,265 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:42,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:42,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T04:41:42,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T04:41:42,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:42,418 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T04:41:42,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/29056fd11b414ca882bcaaf53f30554e is 50, key is test_row_1/A:col10/1731645700812/Put/seqid=0 2024-11-15T04:41:42,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742285_1461 (size=9657) 2024-11-15T04:41:42,560 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T04:41:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T04:41:42,846 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/29056fd11b414ca882bcaaf53f30554e 2024-11-15T04:41:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/9aca64d41ad9402983440013e6c251ac is 50, key is test_row_1/B:col10/1731645700812/Put/seqid=0 2024-11-15T04:41:42,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742286_1462 (size=9657) 2024-11-15T04:41:42,858 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/9aca64d41ad9402983440013e6c251ac 2024-11-15T04:41:42,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/e948e1c0d13a45b99a6240a4d7391dc0 is 50, key is test_row_1/C:col10/1731645700812/Put/seqid=0 2024-11-15T04:41:42,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742287_1463 (size=9657) 2024-11-15T04:41:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T04:41:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:42,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:43,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645762999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645763000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645763005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645763006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645763007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645763113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,118 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645763123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645763123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645763123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645763123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,270 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/e948e1c0d13a45b99a6240a4d7391dc0 2024-11-15T04:41:43,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/29056fd11b414ca882bcaaf53f30554e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/29056fd11b414ca882bcaaf53f30554e 2024-11-15T04:41:43,276 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/29056fd11b414ca882bcaaf53f30554e, entries=100, sequenceid=50, filesize=9.4 K 2024-11-15T04:41:43,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/9aca64d41ad9402983440013e6c251ac as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9aca64d41ad9402983440013e6c251ac 2024-11-15T04:41:43,280 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9aca64d41ad9402983440013e6c251ac, entries=100, sequenceid=50, filesize=9.4 K 2024-11-15T04:41:43,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/e948e1c0d13a45b99a6240a4d7391dc0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/e948e1c0d13a45b99a6240a4d7391dc0 2024-11-15T04:41:43,283 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/e948e1c0d13a45b99a6240a4d7391dc0, entries=100, sequenceid=50, filesize=9.4 K 2024-11-15T04:41:43,283 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c079ffac046c5f34a718ffcef10b362b in 865ms, sequenceid=50, compaction requested=true 2024-11-15T04:41:43,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:43,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:43,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-15T04:41:43,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-15T04:41:43,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-15T04:41:43,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0200 sec 2024-11-15T04:41:43,287 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.0230 sec 2024-11-15T04:41:43,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:43,327 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:43,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T04:41:43,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/320bca2ad112486c8d2d4381394f7f2e is 50, key is test_row_0/A:col10/1731645702998/Put/seqid=0 2024-11-15T04:41:43,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742288_1464 (size=14341) 2024-11-15T04:41:43,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/320bca2ad112486c8d2d4381394f7f2e 2024-11-15T04:41:43,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/61601eb3f0104e38ae98ce3f80b54cca is 50, key is test_row_0/B:col10/1731645702998/Put/seqid=0 2024-11-15T04:41:43,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742289_1465 (size=12001) 2024-11-15T04:41:43,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645763341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645763341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645763344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645763347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645763354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T04:41:43,403 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:43,404 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:43,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-15T04:41:43,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T04:41:43,405 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:43,406 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:43,406 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:43,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645763462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645763463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645763463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645763463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645763472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,479 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T04:41:43,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T04:41:43,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:43,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:43,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645763673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645763673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645763673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645763673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645763682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T04:41:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:43,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T04:41:43,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/61601eb3f0104e38ae98ce3f80b54cca 2024-11-15T04:41:43,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/fd019617b3534df5aacfde665b55c6d5 is 50, key is test_row_0/C:col10/1731645702998/Put/seqid=0 2024-11-15T04:41:43,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742290_1466 (size=12001) 2024-11-15T04:41:43,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T04:41:43,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:43,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:43,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:43,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:43,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:43,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:43,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645763983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645763983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645763983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645763983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:43,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645763992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T04:41:44,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:44,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:44,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:44,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:44,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:44,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T04:41:44,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/fd019617b3534df5aacfde665b55c6d5 2024-11-15T04:41:44,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/320bca2ad112486c8d2d4381394f7f2e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/320bca2ad112486c8d2d4381394f7f2e 2024-11-15T04:41:44,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/320bca2ad112486c8d2d4381394f7f2e, entries=200, sequenceid=74, filesize=14.0 K 2024-11-15T04:41:44,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/61601eb3f0104e38ae98ce3f80b54cca as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/61601eb3f0104e38ae98ce3f80b54cca 2024-11-15T04:41:44,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T04:41:44,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:44,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:44,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:44,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:44,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:44,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/61601eb3f0104e38ae98ce3f80b54cca, entries=150, sequenceid=74, filesize=11.7 K 2024-11-15T04:41:44,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/fd019617b3534df5aacfde665b55c6d5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/fd019617b3534df5aacfde665b55c6d5 2024-11-15T04:41:44,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/fd019617b3534df5aacfde665b55c6d5, entries=150, sequenceid=74, filesize=11.7 K 2024-11-15T04:41:44,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c079ffac046c5f34a718ffcef10b362b in 851ms, sequenceid=74, compaction requested=true 2024-11-15T04:41:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:44,178 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:44,178 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:44,179 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48000 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:44,179 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/A is initiating minor compaction (all files) 2024-11-15T04:41:44,179 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/A in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:44,179 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/981d7de07cd145cebeaf188b05a4d9a1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/83a1a6e92bdb481584a35dacc778b688, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/29056fd11b414ca882bcaaf53f30554e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/320bca2ad112486c8d2d4381394f7f2e] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=46.9 K 2024-11-15T04:41:44,180 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:44,180 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/B is initiating minor compaction (all files) 2024-11-15T04:41:44,180 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 981d7de07cd145cebeaf188b05a4d9a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731645700128 2024-11-15T04:41:44,180 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/B in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:44,180 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/5e43d962cc9a4c09a5c48becf4fca97f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/39875920814b49c5abc259fa7b3bb6df, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9aca64d41ad9402983440013e6c251ac, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/61601eb3f0104e38ae98ce3f80b54cca] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=44.6 K 2024-11-15T04:41:44,180 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83a1a6e92bdb481584a35dacc778b688, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731645700149 2024-11-15T04:41:44,180 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e43d962cc9a4c09a5c48becf4fca97f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731645700128 2024-11-15T04:41:44,180 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 39875920814b49c5abc259fa7b3bb6df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731645700149 2024-11-15T04:41:44,180 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29056fd11b414ca882bcaaf53f30554e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731645700812 2024-11-15T04:41:44,180 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aca64d41ad9402983440013e6c251ac, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731645700812 2024-11-15T04:41:44,180 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 320bca2ad112486c8d2d4381394f7f2e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731645702998 2024-11-15T04:41:44,181 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 61601eb3f0104e38ae98ce3f80b54cca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731645702998 2024-11-15T04:41:44,187 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:44,188 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/baf735d553ad4df7a32736aecbc59f15 is 50, key is test_row_0/B:col10/1731645702998/Put/seqid=0 2024-11-15T04:41:44,188 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#A#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:44,189 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/6cb3b5170f53428da7a1455156e1373a is 50, key is test_row_0/A:col10/1731645702998/Put/seqid=0 2024-11-15T04:41:44,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742291_1467 (size=12139) 2024-11-15T04:41:44,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742292_1468 (size=12139) 2024-11-15T04:41:44,196 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/baf735d553ad4df7a32736aecbc59f15 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/baf735d553ad4df7a32736aecbc59f15 2024-11-15T04:41:44,198 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/6cb3b5170f53428da7a1455156e1373a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/6cb3b5170f53428da7a1455156e1373a 2024-11-15T04:41:44,200 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/B of c079ffac046c5f34a718ffcef10b362b into baf735d553ad4df7a32736aecbc59f15(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:44,200 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:44,200 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/B, priority=12, startTime=1731645704178; duration=0sec 2024-11-15T04:41:44,200 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:44,200 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:B 2024-11-15T04:41:44,200 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:44,202 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:44,202 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/C is initiating minor compaction (all files) 2024-11-15T04:41:44,202 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/C in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:44,202 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/0fe657d52a834fd4964b983cd07c75e8, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/bdb6fb122ba04e049474683431d1a2e4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/e948e1c0d13a45b99a6240a4d7391dc0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/fd019617b3534df5aacfde665b55c6d5] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=44.6 K 2024-11-15T04:41:44,202 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fe657d52a834fd4964b983cd07c75e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731645700128 2024-11-15T04:41:44,202 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb6fb122ba04e049474683431d1a2e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731645700149 2024-11-15T04:41:44,202 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e948e1c0d13a45b99a6240a4d7391dc0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731645700812 2024-11-15T04:41:44,203 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting fd019617b3534df5aacfde665b55c6d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731645702998 2024-11-15T04:41:44,203 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/A of c079ffac046c5f34a718ffcef10b362b into 6cb3b5170f53428da7a1455156e1373a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:44,203 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:44,203 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/A, priority=12, startTime=1731645704178; duration=0sec 2024-11-15T04:41:44,203 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:44,203 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:A 2024-11-15T04:41:44,213 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#C#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:44,213 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/c3fbb49656024ae2b0f2d8f23abda4dc is 50, key is test_row_0/C:col10/1731645702998/Put/seqid=0 2024-11-15T04:41:44,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742293_1469 (size=12139) 2024-11-15T04:41:44,222 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/c3fbb49656024ae2b0f2d8f23abda4dc as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c3fbb49656024ae2b0f2d8f23abda4dc 2024-11-15T04:41:44,228 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/C of c079ffac046c5f34a718ffcef10b362b into c3fbb49656024ae2b0f2d8f23abda4dc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:44,228 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:44,228 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/C, priority=12, startTime=1731645704178; duration=0sec 2024-11-15T04:41:44,228 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:44,228 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:C 2024-11-15T04:41:44,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T04:41:44,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:44,322 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T04:41:44,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/f001eb0909ae42b88096bf7f39f98cf6 is 50, key is test_row_0/A:col10/1731645703342/Put/seqid=0 2024-11-15T04:41:44,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742294_1470 (size=12001) 2024-11-15T04:41:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:44,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:44,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645764527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645764528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645764529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645764530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645764534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T04:41:44,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645764642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645764643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645764643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645764643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645764643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,728 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/f001eb0909ae42b88096bf7f39f98cf6 2024-11-15T04:41:44,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/2aab2d1b2ce94a64a584a67a7042d5eb is 50, key is test_row_0/B:col10/1731645703342/Put/seqid=0 2024-11-15T04:41:44,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742295_1471 (size=12001) 2024-11-15T04:41:44,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645764853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645764853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645764853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645764853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645764853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,138 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/2aab2d1b2ce94a64a584a67a7042d5eb 2024-11-15T04:41:45,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/867238d5a754479b86e7640befb70dd2 is 50, key is test_row_0/C:col10/1731645703342/Put/seqid=0 2024-11-15T04:41:45,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742296_1472 (size=12001) 2024-11-15T04:41:45,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645765162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645765162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645765172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645765172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645765173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,180 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,548 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/867238d5a754479b86e7640befb70dd2 2024-11-15T04:41:45,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/f001eb0909ae42b88096bf7f39f98cf6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/f001eb0909ae42b88096bf7f39f98cf6 2024-11-15T04:41:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T04:41:45,555 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/f001eb0909ae42b88096bf7f39f98cf6, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T04:41:45,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/2aab2d1b2ce94a64a584a67a7042d5eb as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2aab2d1b2ce94a64a584a67a7042d5eb 2024-11-15T04:41:45,558 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2aab2d1b2ce94a64a584a67a7042d5eb, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T04:41:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/867238d5a754479b86e7640befb70dd2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/867238d5a754479b86e7640befb70dd2 2024-11-15T04:41:45,563 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/867238d5a754479b86e7640befb70dd2, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T04:41:45,563 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c079ffac046c5f34a718ffcef10b362b in 1241ms, sequenceid=89, compaction requested=false 2024-11-15T04:41:45,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:45,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:45,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-15T04:41:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-15T04:41:45,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-15T04:41:45,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1590 sec 2024-11-15T04:41:45,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.1630 sec 2024-11-15T04:41:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:45,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:45,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:41:45,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/0aa824e66c1a4084a91e39f9400ce65b is 50, key is test_row_0/A:col10/1731645704519/Put/seqid=0 2024-11-15T04:41:45,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742297_1473 (size=16681) 2024-11-15T04:41:45,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645765693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645765693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645765695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645765696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645765696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645765803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645765803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:45,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645765812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:45,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645766013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645766013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645766023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/0aa824e66c1a4084a91e39f9400ce65b 2024-11-15T04:41:46,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/31fe7ac5d16d4053af4cabde9fb0c980 is 50, key is test_row_0/B:col10/1731645704519/Put/seqid=0 2024-11-15T04:41:46,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742298_1474 (size=12001) 2024-11-15T04:41:46,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645766322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645766323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645766343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/31fe7ac5d16d4053af4cabde9fb0c980 2024-11-15T04:41:46,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/d705e86be92946afb4cf99ed933640c9 is 50, key is test_row_0/C:col10/1731645704519/Put/seqid=0 2024-11-15T04:41:46,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742299_1475 (size=12001) 2024-11-15T04:41:46,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645766703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645766703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,710 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in c079ffac046c5f34a718ffcef10b362b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:41:46,710 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in c079ffac046c5f34a718ffcef10b362b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:41:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645766833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645766843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645766862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:46,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/d705e86be92946afb4cf99ed933640c9 2024-11-15T04:41:46,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/0aa824e66c1a4084a91e39f9400ce65b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0aa824e66c1a4084a91e39f9400ce65b 2024-11-15T04:41:46,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0aa824e66c1a4084a91e39f9400ce65b, entries=250, sequenceid=115, filesize=16.3 K 2024-11-15T04:41:46,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/31fe7ac5d16d4053af4cabde9fb0c980 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/31fe7ac5d16d4053af4cabde9fb0c980 2024-11-15T04:41:46,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/31fe7ac5d16d4053af4cabde9fb0c980, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T04:41:46,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/d705e86be92946afb4cf99ed933640c9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d705e86be92946afb4cf99ed933640c9 2024-11-15T04:41:46,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d705e86be92946afb4cf99ed933640c9, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T04:41:46,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c079ffac046c5f34a718ffcef10b362b in 1242ms, sequenceid=115, compaction requested=true 2024-11-15T04:41:46,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:46,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:46,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:46,919 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:46,919 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:46,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:46,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:46,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:46,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:46,925 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:46,925 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:46,925 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/B is initiating minor compaction (all files) 2024-11-15T04:41:46,925 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/A is initiating minor compaction (all files) 2024-11-15T04:41:46,925 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/B in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:46,925 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/A in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:46,926 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/baf735d553ad4df7a32736aecbc59f15, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2aab2d1b2ce94a64a584a67a7042d5eb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/31fe7ac5d16d4053af4cabde9fb0c980] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=35.3 K 2024-11-15T04:41:46,926 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/6cb3b5170f53428da7a1455156e1373a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/f001eb0909ae42b88096bf7f39f98cf6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0aa824e66c1a4084a91e39f9400ce65b] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=39.9 K 2024-11-15T04:41:46,926 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting baf735d553ad4df7a32736aecbc59f15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731645702998 2024-11-15T04:41:46,926 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cb3b5170f53428da7a1455156e1373a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731645702998 2024-11-15T04:41:46,926 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting f001eb0909ae42b88096bf7f39f98cf6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731645703342 2024-11-15T04:41:46,926 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aab2d1b2ce94a64a584a67a7042d5eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731645703342 2024-11-15T04:41:46,926 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa824e66c1a4084a91e39f9400ce65b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731645704519 2024-11-15T04:41:46,926 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 31fe7ac5d16d4053af4cabde9fb0c980, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731645704519 2024-11-15T04:41:46,932 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#B#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:46,933 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#A#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:46,933 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/dbec1025489045fc9a6ff665563258a9 is 50, key is test_row_0/B:col10/1731645704519/Put/seqid=0 2024-11-15T04:41:46,933 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/1fe071405bba47fa9314ea376a51bb45 is 50, key is test_row_0/A:col10/1731645704519/Put/seqid=0 2024-11-15T04:41:46,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742301_1477 (size=12241) 2024-11-15T04:41:46,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742300_1476 (size=12241) 2024-11-15T04:41:47,360 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/dbec1025489045fc9a6ff665563258a9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/dbec1025489045fc9a6ff665563258a9 2024-11-15T04:41:47,360 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/1fe071405bba47fa9314ea376a51bb45 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/1fe071405bba47fa9314ea376a51bb45 2024-11-15T04:41:47,364 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/B of c079ffac046c5f34a718ffcef10b362b into dbec1025489045fc9a6ff665563258a9(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:47,364 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:47,364 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/B, priority=13, startTime=1731645706919; duration=0sec 2024-11-15T04:41:47,365 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:47,365 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:B 2024-11-15T04:41:47,365 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/A of c079ffac046c5f34a718ffcef10b362b into 1fe071405bba47fa9314ea376a51bb45(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:47,365 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:47,365 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:47,365 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/A, priority=13, startTime=1731645706919; duration=0sec 2024-11-15T04:41:47,365 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:47,365 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:A 2024-11-15T04:41:47,366 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:47,366 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/C is initiating minor compaction (all files) 2024-11-15T04:41:47,366 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/C in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:47,366 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c3fbb49656024ae2b0f2d8f23abda4dc, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/867238d5a754479b86e7640befb70dd2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d705e86be92946afb4cf99ed933640c9] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=35.3 K 2024-11-15T04:41:47,366 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c3fbb49656024ae2b0f2d8f23abda4dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731645702998 2024-11-15T04:41:47,366 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 867238d5a754479b86e7640befb70dd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731645703342 2024-11-15T04:41:47,367 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting d705e86be92946afb4cf99ed933640c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731645704519 2024-11-15T04:41:47,371 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#C#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:47,371 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/923203ce2a8a490f96bc46f27e9f0e13 is 50, key is test_row_0/C:col10/1731645704519/Put/seqid=0 2024-11-15T04:41:47,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742302_1478 (size=12241) 2024-11-15T04:41:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T04:41:47,572 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:47,573 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-15T04:41:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T04:41:47,575 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:47,575 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:47,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T04:41:47,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T04:41:47,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:47,728 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T04:41:47,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/5534053f3a9748369f49c4bd1c057768 is 50, key is test_row_0/A:col10/1731645705695/Put/seqid=0 2024-11-15T04:41:47,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742303_1479 (size=12001) 2024-11-15T04:41:47,778 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/923203ce2a8a490f96bc46f27e9f0e13 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/923203ce2a8a490f96bc46f27e9f0e13 2024-11-15T04:41:47,782 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/C of c079ffac046c5f34a718ffcef10b362b into 923203ce2a8a490f96bc46f27e9f0e13(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:47,782 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:47,782 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/C, priority=13, startTime=1731645706919; duration=0sec 2024-11-15T04:41:47,782 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:47,782 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:C 2024-11-15T04:41:47,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:47,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T04:41:47,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645767921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:47,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:47,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:47,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645767924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:47,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645767926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:47,932 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645768033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645768033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645768042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,135 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/5534053f3a9748369f49c4bd1c057768 2024-11-15T04:41:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/0da6ab77efc24ecfbb1b7c75a6981a61 is 50, key is test_row_0/B:col10/1731645705695/Put/seqid=0 2024-11-15T04:41:48,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742304_1480 (size=12001) 2024-11-15T04:41:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T04:41:48,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645768243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645768243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645768252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,545 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/0da6ab77efc24ecfbb1b7c75a6981a61 2024-11-15T04:41:48,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/5507fe45fe2f4911a67663bcd0663301 is 50, key is test_row_0/C:col10/1731645705695/Put/seqid=0 2024-11-15T04:41:48,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742305_1481 (size=12001) 2024-11-15T04:41:48,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645768552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645768553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645768563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,570 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T04:41:48,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645768722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,728 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in c079ffac046c5f34a718ffcef10b362b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:41:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645768732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:48,738 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in c079ffac046c5f34a718ffcef10b362b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:41:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:48,954 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/5507fe45fe2f4911a67663bcd0663301 2024-11-15T04:41:48,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/5534053f3a9748369f49c4bd1c057768 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5534053f3a9748369f49c4bd1c057768 2024-11-15T04:41:48,961 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5534053f3a9748369f49c4bd1c057768, entries=150, sequenceid=127, filesize=11.7 K 2024-11-15T04:41:48,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/0da6ab77efc24ecfbb1b7c75a6981a61 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/0da6ab77efc24ecfbb1b7c75a6981a61 2024-11-15T04:41:48,964 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/0da6ab77efc24ecfbb1b7c75a6981a61, entries=150, sequenceid=127, filesize=11.7 K 2024-11-15T04:41:48,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/5507fe45fe2f4911a67663bcd0663301 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/5507fe45fe2f4911a67663bcd0663301 2024-11-15T04:41:48,968 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/5507fe45fe2f4911a67663bcd0663301, entries=150, sequenceid=127, filesize=11.7 K 2024-11-15T04:41:48,968 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for c079ffac046c5f34a718ffcef10b362b in 1241ms, sequenceid=127, compaction requested=false 2024-11-15T04:41:48,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:48,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:48,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-15T04:41:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-15T04:41:48,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-15T04:41:48,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3950 sec 2024-11-15T04:41:48,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.3990 sec 2024-11-15T04:41:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:49,071 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:49,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T04:41:49,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/5a6b4c9c3b6945b58729f70c0638391f is 50, key is test_row_0/A:col10/1731645707923/Put/seqid=0 2024-11-15T04:41:49,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742306_1482 (size=14541) 2024-11-15T04:41:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645769083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645769086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645769090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645769192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645769202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645769202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645769403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645769412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645769412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/5a6b4c9c3b6945b58729f70c0638391f 2024-11-15T04:41:49,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/1b9fb4a2ec9c46bbb7491528e4658e25 is 50, key is test_row_0/B:col10/1731645707923/Put/seqid=0 2024-11-15T04:41:49,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742307_1483 (size=12151) 2024-11-15T04:41:49,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/1b9fb4a2ec9c46bbb7491528e4658e25 2024-11-15T04:41:49,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/4f00091a7f9c43ee88533078348baff2 is 50, key is test_row_0/C:col10/1731645707923/Put/seqid=0 2024-11-15T04:41:49,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742308_1484 (size=12151) 2024-11-15T04:41:49,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/4f00091a7f9c43ee88533078348baff2 2024-11-15T04:41:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/5a6b4c9c3b6945b58729f70c0638391f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5a6b4c9c3b6945b58729f70c0638391f 2024-11-15T04:41:49,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5a6b4c9c3b6945b58729f70c0638391f, entries=200, sequenceid=155, filesize=14.2 K 2024-11-15T04:41:49,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/1b9fb4a2ec9c46bbb7491528e4658e25 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/1b9fb4a2ec9c46bbb7491528e4658e25 2024-11-15T04:41:49,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/1b9fb4a2ec9c46bbb7491528e4658e25, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T04:41:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/4f00091a7f9c43ee88533078348baff2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/4f00091a7f9c43ee88533078348baff2 2024-11-15T04:41:49,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/4f00091a7f9c43ee88533078348baff2, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T04:41:49,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for c079ffac046c5f34a718ffcef10b362b in 448ms, sequenceid=155, compaction requested=true 2024-11-15T04:41:49,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:49,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:49,519 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:49,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:49,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:49,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:49,519 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:49,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:49,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:49,520 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:49,520 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:49,520 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/B is initiating minor compaction (all files) 2024-11-15T04:41:49,520 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/A is initiating minor compaction (all files) 2024-11-15T04:41:49,520 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/B in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:49,520 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/A in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:49,520 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/dbec1025489045fc9a6ff665563258a9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/0da6ab77efc24ecfbb1b7c75a6981a61, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/1b9fb4a2ec9c46bbb7491528e4658e25] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=35.5 K 2024-11-15T04:41:49,520 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/1fe071405bba47fa9314ea376a51bb45, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5534053f3a9748369f49c4bd1c057768, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5a6b4c9c3b6945b58729f70c0638391f] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=37.9 K 2024-11-15T04:41:49,520 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting dbec1025489045fc9a6ff665563258a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731645704519 2024-11-15T04:41:49,520 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fe071405bba47fa9314ea376a51bb45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731645704519 2024-11-15T04:41:49,520 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 0da6ab77efc24ecfbb1b7c75a6981a61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731645705687 2024-11-15T04:41:49,520 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5534053f3a9748369f49c4bd1c057768, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731645705687 2024-11-15T04:41:49,521 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9fb4a2ec9c46bbb7491528e4658e25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731645707919 2024-11-15T04:41:49,521 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a6b4c9c3b6945b58729f70c0638391f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731645707919 2024-11-15T04:41:49,526 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#A#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:49,527 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#B#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:49,527 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/7926ae7b9012426a9416395e37c030f3 is 50, key is test_row_0/A:col10/1731645707923/Put/seqid=0 2024-11-15T04:41:49,527 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/b4df328765954726a7d7717bd0f814b2 is 50, key is test_row_0/B:col10/1731645707923/Put/seqid=0 2024-11-15T04:41:49,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742309_1485 (size=12493) 2024-11-15T04:41:49,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742310_1486 (size=12493) 2024-11-15T04:41:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:49,720 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:49,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:41:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T04:41:49,723 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:49,724 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-15T04:41:49,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/be7679febdad4e13b59e56cfd66b632a is 50, key is test_row_0/A:col10/1731645709716/Put/seqid=0 2024-11-15T04:41:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T04:41:49,726 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:49,726 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:49,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:49,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742311_1487 (size=14541) 2024-11-15T04:41:49,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645769786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645769786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645769797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T04:41:49,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T04:41:49,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:49,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:49,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:49,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:49,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:49,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645769902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645769902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645769913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:49,940 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/7926ae7b9012426a9416395e37c030f3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/7926ae7b9012426a9416395e37c030f3 2024-11-15T04:41:49,944 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/A of c079ffac046c5f34a718ffcef10b362b into 7926ae7b9012426a9416395e37c030f3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:49,944 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:49,944 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/A, priority=13, startTime=1731645709519; duration=0sec 2024-11-15T04:41:49,944 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:49,944 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:A 2024-11-15T04:41:49,944 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:49,945 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:49,945 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/C is initiating minor compaction (all files) 2024-11-15T04:41:49,945 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/C in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:49,945 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/923203ce2a8a490f96bc46f27e9f0e13, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/5507fe45fe2f4911a67663bcd0663301, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/4f00091a7f9c43ee88533078348baff2] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=35.5 K 2024-11-15T04:41:49,945 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 923203ce2a8a490f96bc46f27e9f0e13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731645704519 2024-11-15T04:41:49,945 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5507fe45fe2f4911a67663bcd0663301, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731645705687 2024-11-15T04:41:49,946 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f00091a7f9c43ee88533078348baff2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731645707919 2024-11-15T04:41:49,946 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/b4df328765954726a7d7717bd0f814b2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/b4df328765954726a7d7717bd0f814b2 2024-11-15T04:41:49,950 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/B of c079ffac046c5f34a718ffcef10b362b into b4df328765954726a7d7717bd0f814b2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:49,950 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:49,950 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/B, priority=13, startTime=1731645709519; duration=0sec 2024-11-15T04:41:49,950 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:49,950 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:B 2024-11-15T04:41:49,951 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:49,951 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/f7515308172541ef9aa98999ba059e38 is 50, key is test_row_0/C:col10/1731645707923/Put/seqid=0 2024-11-15T04:41:49,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742312_1488 (size=12493) 2024-11-15T04:41:50,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T04:41:50,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:50,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T04:41:50,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645770113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,117 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:50,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645770122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:50,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/be7679febdad4e13b59e56cfd66b632a 2024-11-15T04:41:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645770132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:50,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/fe232f01ca2243e6be17e896002d4069 is 50, key is test_row_0/B:col10/1731645709716/Put/seqid=0 2024-11-15T04:41:50,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742313_1489 (size=12151) 2024-11-15T04:41:50,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T04:41:50,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:50,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T04:41:50,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:50,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T04:41:50,359 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/f7515308172541ef9aa98999ba059e38 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/f7515308172541ef9aa98999ba059e38 2024-11-15T04:41:50,363 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/C of c079ffac046c5f34a718ffcef10b362b into f7515308172541ef9aa98999ba059e38(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:50,363 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:50,363 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/C, priority=13, startTime=1731645709519; duration=0sec 2024-11-15T04:41:50,363 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:50,363 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:C 2024-11-15T04:41:50,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:50,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645770423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:50,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645770433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:50,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645770443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:50,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T04:41:50,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:50,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/fe232f01ca2243e6be17e896002d4069 2024-11-15T04:41:50,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/d80f21df915e4079ac12b5d5ab33467e is 50, key is test_row_0/C:col10/1731645709716/Put/seqid=0 2024-11-15T04:41:50,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742314_1490 (size=12151) 2024-11-15T04:41:50,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T04:41:50,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:50,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T04:41:50,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:50,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T04:41:50,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645770933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:50,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645770943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:50,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T04:41:50,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:50,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:50,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:50,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/d80f21df915e4079ac12b5d5ab33467e 2024-11-15T04:41:50,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/be7679febdad4e13b59e56cfd66b632a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/be7679febdad4e13b59e56cfd66b632a 2024-11-15T04:41:50,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645770952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:50,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:50,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/be7679febdad4e13b59e56cfd66b632a, entries=200, sequenceid=166, filesize=14.2 K 2024-11-15T04:41:50,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/fe232f01ca2243e6be17e896002d4069 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/fe232f01ca2243e6be17e896002d4069 2024-11-15T04:41:50,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/fe232f01ca2243e6be17e896002d4069, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T04:41:50,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/d80f21df915e4079ac12b5d5ab33467e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d80f21df915e4079ac12b5d5ab33467e 2024-11-15T04:41:50,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d80f21df915e4079ac12b5d5ab33467e, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T04:41:50,964 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c079ffac046c5f34a718ffcef10b362b in 1244ms, sequenceid=166, compaction requested=false 2024-11-15T04:41:50,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:51,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T04:41:51,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:51,099 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:41:51,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/4375ebf39029483eaa46a0a9a1986f23 is 50, key is test_row_0/A:col10/1731645709774/Put/seqid=0 2024-11-15T04:41:51,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742315_1491 (size=12151) 2024-11-15T04:41:51,506 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/4375ebf39029483eaa46a0a9a1986f23 2024-11-15T04:41:51,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/9c7d5746e636401ea8d82aa77f475ea1 is 50, key is test_row_0/B:col10/1731645709774/Put/seqid=0 2024-11-15T04:41:51,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742316_1492 (size=12151) 2024-11-15T04:41:51,517 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/9c7d5746e636401ea8d82aa77f475ea1 2024-11-15T04:41:51,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/b66efea2a1534fe88c6600c8600b5dd4 is 50, key is test_row_0/C:col10/1731645709774/Put/seqid=0 2024-11-15T04:41:51,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742317_1493 (size=12151) 2024-11-15T04:41:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T04:41:51,925 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/b66efea2a1534fe88c6600c8600b5dd4 2024-11-15T04:41:51,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/4375ebf39029483eaa46a0a9a1986f23 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/4375ebf39029483eaa46a0a9a1986f23 2024-11-15T04:41:51,932 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/4375ebf39029483eaa46a0a9a1986f23, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T04:41:51,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/9c7d5746e636401ea8d82aa77f475ea1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9c7d5746e636401ea8d82aa77f475ea1 2024-11-15T04:41:51,937 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9c7d5746e636401ea8d82aa77f475ea1, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T04:41:51,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/b66efea2a1534fe88c6600c8600b5dd4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/b66efea2a1534fe88c6600c8600b5dd4 2024-11-15T04:41:51,941 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/b66efea2a1534fe88c6600c8600b5dd4, entries=150, sequenceid=194, filesize=11.9 K 2024-11-15T04:41:51,941 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for c079ffac046c5f34a718ffcef10b362b in 842ms, sequenceid=194, compaction requested=true 2024-11-15T04:41:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:51,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-15T04:41:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-15T04:41:51,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-15T04:41:51,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2170 sec 2024-11-15T04:41:51,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.2210 sec 2024-11-15T04:41:52,000 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:52,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:41:52,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/8d973323ff6e40498cdc448b0871a616 is 50, key is test_row_0/A:col10/1731645711999/Put/seqid=0 2024-11-15T04:41:52,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742318_1494 (size=16931) 2024-11-15T04:41:52,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645772055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645772058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,072 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645772065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645772172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645772182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645772182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645772383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645772392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645772392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,402 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/8d973323ff6e40498cdc448b0871a616 2024-11-15T04:41:52,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/2ec005217873417d800d44e3299247b6 is 50, key is test_row_0/B:col10/1731645711999/Put/seqid=0 2024-11-15T04:41:52,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742319_1495 (size=12151) 2024-11-15T04:41:52,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645772703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645772713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645772713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41484 deadline: 1731645772742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,750 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in c079ffac046c5f34a718ffcef10b362b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:41:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41526 deadline: 1731645772773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:52,781 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in c079ffac046c5f34a718ffcef10b362b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:41:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:52,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:52,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/2ec005217873417d800d44e3299247b6 2024-11-15T04:41:52,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/18cf58dc3ea245aeb59793914dc02e4b is 50, key is test_row_0/C:col10/1731645711999/Put/seqid=0 2024-11-15T04:41:52,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742320_1496 (size=12151) 2024-11-15T04:41:52,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/18cf58dc3ea245aeb59793914dc02e4b 2024-11-15T04:41:52,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/8d973323ff6e40498cdc448b0871a616 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/8d973323ff6e40498cdc448b0871a616 2024-11-15T04:41:52,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/8d973323ff6e40498cdc448b0871a616, entries=250, sequenceid=206, filesize=16.5 K 2024-11-15T04:41:52,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/2ec005217873417d800d44e3299247b6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2ec005217873417d800d44e3299247b6 2024-11-15T04:41:52,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2ec005217873417d800d44e3299247b6, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T04:41:52,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/18cf58dc3ea245aeb59793914dc02e4b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/18cf58dc3ea245aeb59793914dc02e4b 2024-11-15T04:41:52,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/18cf58dc3ea245aeb59793914dc02e4b, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T04:41:52,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c079ffac046c5f34a718ffcef10b362b in 881ms, sequenceid=206, compaction requested=true 2024-11-15T04:41:52,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:52,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:52,881 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:52,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:52,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:52,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:52,881 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:52,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:52,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:52,882 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:52,882 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:52,882 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/A is initiating minor compaction (all files) 2024-11-15T04:41:52,882 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/B is initiating minor compaction (all files) 2024-11-15T04:41:52,882 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/A in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:52,882 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/B in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:52,882 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/7926ae7b9012426a9416395e37c030f3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/be7679febdad4e13b59e56cfd66b632a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/4375ebf39029483eaa46a0a9a1986f23, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/8d973323ff6e40498cdc448b0871a616] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=54.8 K 2024-11-15T04:41:52,882 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/b4df328765954726a7d7717bd0f814b2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/fe232f01ca2243e6be17e896002d4069, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9c7d5746e636401ea8d82aa77f475ea1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2ec005217873417d800d44e3299247b6] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=47.8 K 2024-11-15T04:41:52,882 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7926ae7b9012426a9416395e37c030f3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731645707919 2024-11-15T04:41:52,883 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting b4df328765954726a7d7717bd0f814b2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731645707919 2024-11-15T04:41:52,883 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting be7679febdad4e13b59e56cfd66b632a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731645709072 2024-11-15T04:41:52,883 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting fe232f01ca2243e6be17e896002d4069, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731645709072 2024-11-15T04:41:52,883 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4375ebf39029483eaa46a0a9a1986f23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731645709774 2024-11-15T04:41:52,883 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c7d5746e636401ea8d82aa77f475ea1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731645709774 2024-11-15T04:41:52,883 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d973323ff6e40498cdc448b0871a616, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731645711956 2024-11-15T04:41:52,883 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec005217873417d800d44e3299247b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731645711993 2024-11-15T04:41:52,892 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#B#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:52,892 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:52,892 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/66529fb526114b06a6c934b7b2a702c6 is 50, key is test_row_0/B:col10/1731645711999/Put/seqid=0 2024-11-15T04:41:52,892 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/c848939e02df4a15ad4c165d64cbb725 is 50, key is test_row_0/A:col10/1731645711999/Put/seqid=0 2024-11-15T04:41:52,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742322_1498 (size=12629) 2024-11-15T04:41:52,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742321_1497 (size=12629) 2024-11-15T04:41:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:53,218 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:53,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:41:53,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/2126781c46aa4b76a225e9de7c411019 is 50, key is test_row_0/A:col10/1731645713217/Put/seqid=0 2024-11-15T04:41:53,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742323_1499 (size=14541) 2024-11-15T04:41:53,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645773254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645773255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645773256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,303 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/c848939e02df4a15ad4c165d64cbb725 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/c848939e02df4a15ad4c165d64cbb725 2024-11-15T04:41:53,307 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/A of c079ffac046c5f34a718ffcef10b362b into c848939e02df4a15ad4c165d64cbb725(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:53,307 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:53,307 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/A, priority=12, startTime=1731645712881; duration=0sec 2024-11-15T04:41:53,307 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:53,307 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:A 2024-11-15T04:41:53,307 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:53,309 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/66529fb526114b06a6c934b7b2a702c6 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/66529fb526114b06a6c934b7b2a702c6 2024-11-15T04:41:53,310 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:53,310 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/C is initiating minor compaction (all files) 2024-11-15T04:41:53,310 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/C in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:53,310 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/f7515308172541ef9aa98999ba059e38, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d80f21df915e4079ac12b5d5ab33467e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/b66efea2a1534fe88c6600c8600b5dd4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/18cf58dc3ea245aeb59793914dc02e4b] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=47.8 K 2024-11-15T04:41:53,310 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7515308172541ef9aa98999ba059e38, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731645707919 2024-11-15T04:41:53,311 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d80f21df915e4079ac12b5d5ab33467e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731645709072 2024-11-15T04:41:53,311 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b66efea2a1534fe88c6600c8600b5dd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731645709774 2024-11-15T04:41:53,311 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18cf58dc3ea245aeb59793914dc02e4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731645711993 2024-11-15T04:41:53,314 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/B of c079ffac046c5f34a718ffcef10b362b into 66529fb526114b06a6c934b7b2a702c6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:53,314 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:53,314 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/B, priority=12, startTime=1731645712881; duration=0sec 2024-11-15T04:41:53,314 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:53,314 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:B 2024-11-15T04:41:53,321 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#C#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:53,321 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/316bc90230084bfdadbf04e71525e26a is 50, key is test_row_0/C:col10/1731645711999/Put/seqid=0 2024-11-15T04:41:53,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742324_1500 (size=12629) 2024-11-15T04:41:53,328 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/316bc90230084bfdadbf04e71525e26a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/316bc90230084bfdadbf04e71525e26a 2024-11-15T04:41:53,332 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/C of c079ffac046c5f34a718ffcef10b362b into 316bc90230084bfdadbf04e71525e26a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:53,332 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:53,332 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/C, priority=12, startTime=1731645712881; duration=0sec 2024-11-15T04:41:53,332 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:53,332 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:C 2024-11-15T04:41:53,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645773372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645773372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645773372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645773582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645773582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645773582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/2126781c46aa4b76a225e9de7c411019 2024-11-15T04:41:53,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/d1481b9f0c3449eea775eaa3562f306f is 50, key is test_row_0/B:col10/1731645713217/Put/seqid=0 2024-11-15T04:41:53,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742325_1501 (size=12151) 2024-11-15T04:41:53,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645773893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645773893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645773893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:53,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T04:41:53,903 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:53,904 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-15T04:41:53,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T04:41:53,905 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:53,906 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:53,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T04:41:54,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/d1481b9f0c3449eea775eaa3562f306f 2024-11-15T04:41:54,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/7227fdc8d8f4449e92275cbf8a676aad is 50, key is test_row_0/C:col10/1731645713217/Put/seqid=0 2024-11-15T04:41:54,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742326_1502 (size=12151) 2024-11-15T04:41:54,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T04:41:54,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:54,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:54,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:54,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:54,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:54,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T04:41:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:54,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T04:41:54,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T04:41:54,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:54,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:54,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:54,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:54,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645774403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645774403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645774403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:54,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/7227fdc8d8f4449e92275cbf8a676aad 2024-11-15T04:41:54,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/2126781c46aa4b76a225e9de7c411019 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/2126781c46aa4b76a225e9de7c411019 2024-11-15T04:41:54,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/2126781c46aa4b76a225e9de7c411019, entries=200, sequenceid=231, filesize=14.2 K 2024-11-15T04:41:54,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/d1481b9f0c3449eea775eaa3562f306f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/d1481b9f0c3449eea775eaa3562f306f 2024-11-15T04:41:54,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/d1481b9f0c3449eea775eaa3562f306f, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T04:41:54,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/7227fdc8d8f4449e92275cbf8a676aad as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7227fdc8d8f4449e92275cbf8a676aad 2024-11-15T04:41:54,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7227fdc8d8f4449e92275cbf8a676aad, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T04:41:54,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c079ffac046c5f34a718ffcef10b362b in 1258ms, sequenceid=231, compaction requested=false 2024-11-15T04:41:54,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:54,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T04:41:54,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:54,516 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:41:54,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/b5c05537b9fb438bbdfdeedf8a5e785d is 50, key is test_row_0/A:col10/1731645713255/Put/seqid=0 2024-11-15T04:41:54,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742327_1503 (size=12151) 2024-11-15T04:41:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T04:41:54,924 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/b5c05537b9fb438bbdfdeedf8a5e785d 2024-11-15T04:41:54,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/65d608891c6049549feca2d030e57ebb is 50, key is test_row_0/B:col10/1731645713255/Put/seqid=0 2024-11-15T04:41:54,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742328_1504 (size=12151) 2024-11-15T04:41:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T04:41:55,332 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/65d608891c6049549feca2d030e57ebb 2024-11-15T04:41:55,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/a4019f26458b4147ad2522e72ecbdd4c is 50, key is test_row_0/C:col10/1731645713255/Put/seqid=0 2024-11-15T04:41:55,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742329_1505 (size=12151) 2024-11-15T04:41:55,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:55,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:55,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645775465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645775468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645775470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645775572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645775582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645775582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,742 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/a4019f26458b4147ad2522e72ecbdd4c 2024-11-15T04:41:55,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/b5c05537b9fb438bbdfdeedf8a5e785d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/b5c05537b9fb438bbdfdeedf8a5e785d 2024-11-15T04:41:55,749 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/b5c05537b9fb438bbdfdeedf8a5e785d, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T04:41:55,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/65d608891c6049549feca2d030e57ebb as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/65d608891c6049549feca2d030e57ebb 2024-11-15T04:41:55,752 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/65d608891c6049549feca2d030e57ebb, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T04:41:55,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/a4019f26458b4147ad2522e72ecbdd4c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a4019f26458b4147ad2522e72ecbdd4c 2024-11-15T04:41:55,756 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a4019f26458b4147ad2522e72ecbdd4c, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T04:41:55,757 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c079ffac046c5f34a718ffcef10b362b in 1241ms, sequenceid=245, compaction requested=true 2024-11-15T04:41:55,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:55,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:55,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-15T04:41:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-15T04:41:55,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-15T04:41:55,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8520 sec 2024-11-15T04:41:55,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.8560 sec 2024-11-15T04:41:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:55,786 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:55,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T04:41:55,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/ecfc6714c434490dbec6b82f789b0ea5 is 50, key is test_row_0/A:col10/1731645715459/Put/seqid=0 2024-11-15T04:41:55,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742330_1506 (size=14741) 2024-11-15T04:41:55,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645775836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645775836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645775836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645775953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645775953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:55,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645775953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:55,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:55,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:55,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T04:41:56,053 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:56,053 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-15T04:41:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T04:41:56,055 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:56,056 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:56,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T04:41:56,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645776163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645776163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:56,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645776163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:56,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/ecfc6714c434490dbec6b82f789b0ea5 2024-11-15T04:41:56,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/25a2058865024606aed2f23106e529c8 is 50, key is test_row_0/B:col10/1731645715459/Put/seqid=0 2024-11-15T04:41:56,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742331_1507 (size=12301) 2024-11-15T04:41:56,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T04:41:56,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:56,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T04:41:56,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:56,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T04:41:56,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645776473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:56,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645776483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:56,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645776483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:56,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:56,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T04:41:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:56,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/25a2058865024606aed2f23106e529c8 2024-11-15T04:41:56,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/a56621255ed747569dd015bdf31cb4c0 is 50, key is test_row_0/C:col10/1731645715459/Put/seqid=0 2024-11-15T04:41:56,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742332_1508 (size=12301) 2024-11-15T04:41:56,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T04:41:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:56,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T04:41:56,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T04:41:56,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:56,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T04:41:56,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:56,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:56,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:57,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645776993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:57,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:57,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645777003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:57,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:57,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645777003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:57,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:57,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/a56621255ed747569dd015bdf31cb4c0 2024-11-15T04:41:57,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/ecfc6714c434490dbec6b82f789b0ea5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/ecfc6714c434490dbec6b82f789b0ea5 2024-11-15T04:41:57,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/ecfc6714c434490dbec6b82f789b0ea5, entries=200, sequenceid=271, filesize=14.4 K 2024-11-15T04:41:57,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/25a2058865024606aed2f23106e529c8 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/25a2058865024606aed2f23106e529c8 2024-11-15T04:41:57,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/25a2058865024606aed2f23106e529c8, entries=150, sequenceid=271, filesize=12.0 K 2024-11-15T04:41:57,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/a56621255ed747569dd015bdf31cb4c0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a56621255ed747569dd015bdf31cb4c0 2024-11-15T04:41:57,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a56621255ed747569dd015bdf31cb4c0, entries=150, sequenceid=271, filesize=12.0 K 2024-11-15T04:41:57,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for c079ffac046c5f34a718ffcef10b362b in 1247ms, sequenceid=271, compaction requested=true 2024-11-15T04:41:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:57,033 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:57,033 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:57,035 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54062 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:57,035 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/A is initiating minor compaction (all files) 2024-11-15T04:41:57,035 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/A in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:57,035 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/c848939e02df4a15ad4c165d64cbb725, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/2126781c46aa4b76a225e9de7c411019, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/b5c05537b9fb438bbdfdeedf8a5e785d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/ecfc6714c434490dbec6b82f789b0ea5] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=52.8 K 2024-11-15T04:41:57,035 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:57,035 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/B is initiating minor compaction (all files) 2024-11-15T04:41:57,035 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/B in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:57,035 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting c848939e02df4a15ad4c165d64cbb725, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731645711993 2024-11-15T04:41:57,036 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/66529fb526114b06a6c934b7b2a702c6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/d1481b9f0c3449eea775eaa3562f306f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/65d608891c6049549feca2d030e57ebb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/25a2058865024606aed2f23106e529c8] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=48.1 K 2024-11-15T04:41:57,036 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 66529fb526114b06a6c934b7b2a702c6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731645711993 2024-11-15T04:41:57,036 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2126781c46aa4b76a225e9de7c411019, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731645712050 2024-11-15T04:41:57,036 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting d1481b9f0c3449eea775eaa3562f306f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731645712050 2024-11-15T04:41:57,036 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5c05537b9fb438bbdfdeedf8a5e785d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731645713253 2024-11-15T04:41:57,036 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d608891c6049549feca2d030e57ebb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731645713253 2024-11-15T04:41:57,036 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecfc6714c434490dbec6b82f789b0ea5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731645715459 2024-11-15T04:41:57,036 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 25a2058865024606aed2f23106e529c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731645715459 2024-11-15T04:41:57,043 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#B#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:57,044 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/64550e1bf0364d66a3fac7d583097151 is 50, key is test_row_0/B:col10/1731645715459/Put/seqid=0 2024-11-15T04:41:57,047 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#A#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:57,047 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/3c911b1d9d644197980b999900ac8b32 is 50, key is test_row_0/A:col10/1731645715459/Put/seqid=0 2024-11-15T04:41:57,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742333_1509 (size=12915) 2024-11-15T04:41:57,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742334_1510 (size=12915) 2024-11-15T04:41:57,057 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/3c911b1d9d644197980b999900ac8b32 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/3c911b1d9d644197980b999900ac8b32 2024-11-15T04:41:57,061 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/A of c079ffac046c5f34a718ffcef10b362b into 3c911b1d9d644197980b999900ac8b32(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:57,061 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:57,061 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/A, priority=12, startTime=1731645717033; duration=0sec 2024-11-15T04:41:57,061 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:57,061 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:A 2024-11-15T04:41:57,061 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:41:57,062 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:41:57,062 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/C is initiating minor compaction (all files) 2024-11-15T04:41:57,062 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/C in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:57,062 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/316bc90230084bfdadbf04e71525e26a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7227fdc8d8f4449e92275cbf8a676aad, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a4019f26458b4147ad2522e72ecbdd4c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a56621255ed747569dd015bdf31cb4c0] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=48.1 K 2024-11-15T04:41:57,062 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 316bc90230084bfdadbf04e71525e26a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731645711993 2024-11-15T04:41:57,063 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7227fdc8d8f4449e92275cbf8a676aad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731645712050 2024-11-15T04:41:57,063 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4019f26458b4147ad2522e72ecbdd4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731645713253 2024-11-15T04:41:57,063 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56621255ed747569dd015bdf31cb4c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731645715459 2024-11-15T04:41:57,069 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#C#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:57,069 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/c3160b645c4849c78f84d577b815b45e is 50, key is test_row_0/C:col10/1731645715459/Put/seqid=0 2024-11-15T04:41:57,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742335_1511 (size=12915) 2024-11-15T04:41:57,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T04:41:57,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:57,124 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:41:57,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/0ce253eee9384e32afc4cda86a5dce88 is 50, key is test_row_0/A:col10/1731645715819/Put/seqid=0 2024-11-15T04:41:57,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742336_1512 (size=12301) 2024-11-15T04:41:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T04:41:57,486 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/64550e1bf0364d66a3fac7d583097151 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/64550e1bf0364d66a3fac7d583097151 2024-11-15T04:41:57,487 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/c3160b645c4849c78f84d577b815b45e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c3160b645c4849c78f84d577b815b45e 2024-11-15T04:41:57,490 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/B of c079ffac046c5f34a718ffcef10b362b into 64550e1bf0364d66a3fac7d583097151(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:57,490 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:57,490 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/B, priority=12, startTime=1731645717033; duration=0sec 2024-11-15T04:41:57,490 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:57,490 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:B 2024-11-15T04:41:57,491 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/C of c079ffac046c5f34a718ffcef10b362b into c3160b645c4849c78f84d577b815b45e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:57,491 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:57,491 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/C, priority=12, startTime=1731645717033; duration=0sec 2024-11-15T04:41:57,491 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:57,491 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:C 2024-11-15T04:41:57,536 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/0ce253eee9384e32afc4cda86a5dce88 2024-11-15T04:41:57,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/e43978385dcb4384b995c15efc2be4f2 is 50, key is test_row_0/B:col10/1731645715819/Put/seqid=0 2024-11-15T04:41:57,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742337_1513 (size=12301) 2024-11-15T04:41:57,547 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/e43978385dcb4384b995c15efc2be4f2 2024-11-15T04:41:57,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/7c0ecb14db404a9fa69c6348946d95ca is 50, key is test_row_0/C:col10/1731645715819/Put/seqid=0 2024-11-15T04:41:57,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742338_1514 (size=12301) 2024-11-15T04:41:57,957 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/7c0ecb14db404a9fa69c6348946d95ca 2024-11-15T04:41:57,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/0ce253eee9384e32afc4cda86a5dce88 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0ce253eee9384e32afc4cda86a5dce88 2024-11-15T04:41:57,963 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0ce253eee9384e32afc4cda86a5dce88, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T04:41:57,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/e43978385dcb4384b995c15efc2be4f2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/e43978385dcb4384b995c15efc2be4f2 2024-11-15T04:41:57,967 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/e43978385dcb4384b995c15efc2be4f2, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T04:41:57,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/7c0ecb14db404a9fa69c6348946d95ca as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7c0ecb14db404a9fa69c6348946d95ca 2024-11-15T04:41:57,974 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7c0ecb14db404a9fa69c6348946d95ca, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T04:41:57,975 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for c079ffac046c5f34a718ffcef10b362b in 851ms, sequenceid=283, compaction requested=false 2024-11-15T04:41:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-15T04:41:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-15T04:41:57,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-15T04:41:57,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9200 sec 2024-11-15T04:41:57,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9260 sec 2024-11-15T04:41:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:41:58,027 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:41:58,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:41:58,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/d7e3b590ddd14aa0bbd89d513927bcd8 is 50, key is test_row_0/A:col10/1731645718016/Put/seqid=0 2024-11-15T04:41:58,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742339_1515 (size=14741) 2024-11-15T04:41:58,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645778083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645778085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645778088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T04:41:58,203 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:41:58,204 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:41:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-15T04:41:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T04:41:58,205 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:41:58,206 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:41:58,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:41:58,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645778203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645778203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645778203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T04:41:58,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T04:41:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:58,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645778413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645778413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645778413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/d7e3b590ddd14aa0bbd89d513927bcd8 2024-11-15T04:41:58,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/8e085674cb4944d392fb1ac396d40b63 is 50, key is test_row_0/B:col10/1731645718016/Put/seqid=0 2024-11-15T04:41:58,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742340_1516 (size=12301) 2024-11-15T04:41:58,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T04:41:58,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:58,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:58,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:58,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T04:41:58,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T04:41:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:58,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645778722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645778723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645778733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:58,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T04:41:58,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:58,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:58,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:58,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T04:41:58,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/8e085674cb4944d392fb1ac396d40b63 2024-11-15T04:41:58,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/cbe3808df9ab4308a6a4ef0ef84c2280 is 50, key is test_row_0/C:col10/1731645718016/Put/seqid=0 2024-11-15T04:41:58,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742341_1517 (size=12301) 2024-11-15T04:41:58,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T04:41:58,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:58,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:59,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T04:41:59,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:59,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:59,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:59,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:59,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:59,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:59,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41510 deadline: 1731645779233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:59,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41544 deadline: 1731645779243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:59,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:41:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41528 deadline: 1731645779253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:41:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 , the old value is region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:41:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., hostname=26747e6ada4f,42935,1731645581086, seqNum=2 because the exception is null or not the one we care about 2024-11-15T04:41:59,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/cbe3808df9ab4308a6a4ef0ef84c2280 2024-11-15T04:41:59,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/d7e3b590ddd14aa0bbd89d513927bcd8 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/d7e3b590ddd14aa0bbd89d513927bcd8 2024-11-15T04:41:59,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/d7e3b590ddd14aa0bbd89d513927bcd8, entries=200, sequenceid=296, filesize=14.4 K 2024-11-15T04:41:59,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/8e085674cb4944d392fb1ac396d40b63 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/8e085674cb4944d392fb1ac396d40b63 2024-11-15T04:41:59,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/8e085674cb4944d392fb1ac396d40b63, entries=150, sequenceid=296, filesize=12.0 K 2024-11-15T04:41:59,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/cbe3808df9ab4308a6a4ef0ef84c2280 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/cbe3808df9ab4308a6a4ef0ef84c2280 2024-11-15T04:41:59,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T04:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:59,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:41:59,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/cbe3808df9ab4308a6a4ef0ef84c2280, entries=150, sequenceid=296, filesize=12.0 K 2024-11-15T04:41:59,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c079ffac046c5f34a718ffcef10b362b in 1249ms, sequenceid=296, compaction requested=true 2024-11-15T04:41:59,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:59,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:41:59,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:59,277 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:59,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:41:59,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:59,277 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:59,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c079ffac046c5f34a718ffcef10b362b:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:41:59,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:59,277 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:59,277 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/A is initiating minor compaction (all files) 2024-11-15T04:41:59,277 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/A in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:59,277 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/3c911b1d9d644197980b999900ac8b32, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0ce253eee9384e32afc4cda86a5dce88, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/d7e3b590ddd14aa0bbd89d513927bcd8] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=39.0 K 2024-11-15T04:41:59,278 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:59,278 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/B is initiating minor compaction (all files) 2024-11-15T04:41:59,278 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/B in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:59,278 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/64550e1bf0364d66a3fac7d583097151, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/e43978385dcb4384b995c15efc2be4f2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/8e085674cb4944d392fb1ac396d40b63] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=36.6 K 2024-11-15T04:41:59,278 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c911b1d9d644197980b999900ac8b32, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731645715459 2024-11-15T04:41:59,278 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ce253eee9384e32afc4cda86a5dce88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731645715801 2024-11-15T04:41:59,278 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 64550e1bf0364d66a3fac7d583097151, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731645715459 2024-11-15T04:41:59,279 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7e3b590ddd14aa0bbd89d513927bcd8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731645718016 2024-11-15T04:41:59,279 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e43978385dcb4384b995c15efc2be4f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731645715801 2024-11-15T04:41:59,279 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e085674cb4944d392fb1ac396d40b63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731645718016 2024-11-15T04:41:59,285 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#B#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:59,286 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/3d92fe0565f3469b9b09af97105a0607 is 50, key is test_row_0/B:col10/1731645718016/Put/seqid=0 2024-11-15T04:41:59,288 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#A#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:59,288 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/85ae244fa49b4b45be2f64dfa9ebd8ab is 50, key is test_row_0/A:col10/1731645718016/Put/seqid=0 2024-11-15T04:41:59,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742342_1518 (size=13017) 2024-11-15T04:41:59,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742343_1519 (size=13017) 2024-11-15T04:41:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T04:41:59,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T04:41:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:59,426 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:41:59,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/a561a993734142e88827a84a8829f252 is 50, key is test_row_0/A:col10/1731645718070/Put/seqid=0 2024-11-15T04:41:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742344_1520 (size=12301) 2024-11-15T04:41:59,704 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/3d92fe0565f3469b9b09af97105a0607 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/3d92fe0565f3469b9b09af97105a0607 2024-11-15T04:41:59,704 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/85ae244fa49b4b45be2f64dfa9ebd8ab as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/85ae244fa49b4b45be2f64dfa9ebd8ab 2024-11-15T04:41:59,707 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/B of c079ffac046c5f34a718ffcef10b362b into 3d92fe0565f3469b9b09af97105a0607(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:59,707 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/A of c079ffac046c5f34a718ffcef10b362b into 85ae244fa49b4b45be2f64dfa9ebd8ab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:41:59,707 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:59,707 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:41:59,707 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/B, priority=13, startTime=1731645719277; duration=0sec 2024-11-15T04:41:59,707 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/A, priority=13, startTime=1731645719276; duration=0sec 2024-11-15T04:41:59,707 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:41:59,707 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:B 2024-11-15T04:41:59,707 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:41:59,707 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:A 2024-11-15T04:41:59,707 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:41:59,708 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:41:59,708 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): c079ffac046c5f34a718ffcef10b362b/C is initiating minor compaction (all files) 2024-11-15T04:41:59,708 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c079ffac046c5f34a718ffcef10b362b/C in TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:41:59,708 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c3160b645c4849c78f84d577b815b45e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7c0ecb14db404a9fa69c6348946d95ca, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/cbe3808df9ab4308a6a4ef0ef84c2280] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp, totalSize=36.6 K 2024-11-15T04:41:59,709 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c3160b645c4849c78f84d577b815b45e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731645715459 2024-11-15T04:41:59,709 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c0ecb14db404a9fa69c6348946d95ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731645715801 2024-11-15T04:41:59,709 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting cbe3808df9ab4308a6a4ef0ef84c2280, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731645718016 2024-11-15T04:41:59,714 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c079ffac046c5f34a718ffcef10b362b#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:41:59,715 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/b30d3f12cb71487eab4b5b612587fec8 is 50, key is test_row_0/C:col10/1731645718016/Put/seqid=0 2024-11-15T04:41:59,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742345_1521 (size=13017) 2024-11-15T04:41:59,834 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/a561a993734142e88827a84a8829f252 2024-11-15T04:41:59,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/6b662f48e437497bb46dfadfc693c6d2 is 50, key is test_row_0/B:col10/1731645718070/Put/seqid=0 2024-11-15T04:41:59,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742346_1522 (size=12301) 2024-11-15T04:42:00,121 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/b30d3f12cb71487eab4b5b612587fec8 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/b30d3f12cb71487eab4b5b612587fec8 2024-11-15T04:42:00,122 INFO [Thread-2041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2041. 2024-11-15T04:42:00,122 DEBUG [Thread-2041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:00,122 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,122 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:00,123 INFO [Thread-2043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2043. 2024-11-15T04:42:00,123 DEBUG [Thread-2043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:00,123 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,123 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,123 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:00,125 INFO [Thread-2045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2045. 2024-11-15T04:42:00,125 DEBUG [Thread-2045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:00,125 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,125 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:00,125 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c079ffac046c5f34a718ffcef10b362b/C of c079ffac046c5f34a718ffcef10b362b into b30d3f12cb71487eab4b5b612587fec8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:00,125 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:42:00,125 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., storeName=c079ffac046c5f34a718ffcef10b362b/C, priority=13, startTime=1731645719277; duration=0sec 2024-11-15T04:42:00,126 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:00,126 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c079ffac046c5f34a718ffcef10b362b:C 2024-11-15T04:42:00,128 INFO [Thread-2049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2049. 2024-11-15T04:42:00,128 DEBUG [Thread-2049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:00,128 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,128 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,128 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:00,129 INFO [Thread-2047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2047. 2024-11-15T04:42:00,129 DEBUG [Thread-2047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:00,129 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,129 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:00,244 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/6b662f48e437497bb46dfadfc693c6d2 2024-11-15T04:42:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:42:00,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. as already flushing 2024-11-15T04:42:00,255 INFO [Thread-2034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2034. 2024-11-15T04:42:00,255 DEBUG [Thread-2034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:00,255 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,255 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:00,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/82dab27b538b4227818164285e4e9770 is 50, key is test_row_0/C:col10/1731645718070/Put/seqid=0 2024-11-15T04:42:00,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742347_1523 (size=12301) 2024-11-15T04:42:00,264 INFO [Thread-2038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2038. 2024-11-15T04:42:00,264 DEBUG [Thread-2038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:00,264 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,264 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,264 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:00,274 INFO [Thread-2036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2036. 2024-11-15T04:42:00,274 DEBUG [Thread-2036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:00,274 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,274 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:00,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T04:42:00,661 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/82dab27b538b4227818164285e4e9770 2024-11-15T04:42:00,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/a561a993734142e88827a84a8829f252 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/a561a993734142e88827a84a8829f252 2024-11-15T04:42:00,672 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/a561a993734142e88827a84a8829f252, entries=150, sequenceid=321, filesize=12.0 K 2024-11-15T04:42:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/6b662f48e437497bb46dfadfc693c6d2 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/6b662f48e437497bb46dfadfc693c6d2 2024-11-15T04:42:00,676 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/6b662f48e437497bb46dfadfc693c6d2, entries=150, sequenceid=321, filesize=12.0 K 2024-11-15T04:42:00,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/82dab27b538b4227818164285e4e9770 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/82dab27b538b4227818164285e4e9770 2024-11-15T04:42:00,679 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/82dab27b538b4227818164285e4e9770, entries=150, sequenceid=321, filesize=12.0 K 2024-11-15T04:42:00,680 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=20.13 KB/20610 for c079ffac046c5f34a718ffcef10b362b in 1254ms, sequenceid=321, compaction requested=false 2024-11-15T04:42:00,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for c079ffac046c5f34a718ffcef10b362b: 2024-11-15T04:42:00,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:42:00,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-15T04:42:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-15T04:42:00,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-15T04:42:00,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4760 sec 2024-11-15T04:42:00,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.4800 sec 2024-11-15T04:42:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T04:42:02,383 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:02,798 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-11-15T04:42:02,798 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:02,798 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:02,798 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:02,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:02,816 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-11-15T04:42:02,816 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:02,816 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:02,816 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:02,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:02,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T04:42:02,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-15T04:42:02,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-15T04:42:02,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-15T04:42:02,816 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1621 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4853 rows 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1615 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4841 rows 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1613 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4835 rows 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1612 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4831 rows 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1606 2024-11-15T04:42:02,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4814 rows 2024-11-15T04:42:02,817 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T04:42:02,817 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T04:42:02,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:02,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:02,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:02,819 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T04:42:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T04:42:02,824 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645722823"}]},"ts":"1731645722823"} 2024-11-15T04:42:02,825 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T04:42:02,825 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T04:42:02,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T04:42:02,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c079ffac046c5f34a718ffcef10b362b, UNASSIGN}] 2024-11-15T04:42:02,829 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c079ffac046c5f34a718ffcef10b362b, UNASSIGN 2024-11-15T04:42:02,830 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=c079ffac046c5f34a718ffcef10b362b, regionState=CLOSING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:02,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c079ffac046c5f34a718ffcef10b362b, UNASSIGN because future has completed 2024-11-15T04:42:02,831 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T04:42:02,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:42:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T04:42:02,985 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:42:02,985 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T04:42:02,985 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing c079ffac046c5f34a718ffcef10b362b, disabling compactions & flushes 2024-11-15T04:42:02,985 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:42:02,985 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:42:02,985 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. after waiting 0 ms 2024-11-15T04:42:02,985 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:42:02,986 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing c079ffac046c5f34a718ffcef10b362b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T04:42:02,993 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/37ff44e5cc6b46e68a2e893841fa0432 is 50, key is test_row_0/A:col10/1731645722814/Put/seqid=0 2024-11-15T04:42:02,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742348_1524 (size=12301) 2024-11-15T04:42:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T04:42:03,400 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/37ff44e5cc6b46e68a2e893841fa0432 2024-11-15T04:42:03,415 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/4d7cad15a89f4b468b1cbf992a64a3a9 is 50, key is test_row_0/B:col10/1731645722814/Put/seqid=0 2024-11-15T04:42:03,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742349_1525 (size=12301) 2024-11-15T04:42:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T04:42:03,819 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/4d7cad15a89f4b468b1cbf992a64a3a9 2024-11-15T04:42:03,831 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/c9274de5e19f4537bec7af710c4f8c00 is 50, key is test_row_0/C:col10/1731645722814/Put/seqid=0 2024-11-15T04:42:03,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742350_1526 (size=12301) 2024-11-15T04:42:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T04:42:04,237 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/c9274de5e19f4537bec7af710c4f8c00 2024-11-15T04:42:04,247 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/A/37ff44e5cc6b46e68a2e893841fa0432 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/37ff44e5cc6b46e68a2e893841fa0432 2024-11-15T04:42:04,253 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/37ff44e5cc6b46e68a2e893841fa0432, entries=150, sequenceid=332, filesize=12.0 K 2024-11-15T04:42:04,254 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/B/4d7cad15a89f4b468b1cbf992a64a3a9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/4d7cad15a89f4b468b1cbf992a64a3a9 2024-11-15T04:42:04,258 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/4d7cad15a89f4b468b1cbf992a64a3a9, entries=150, sequenceid=332, filesize=12.0 K 2024-11-15T04:42:04,259 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/.tmp/C/c9274de5e19f4537bec7af710c4f8c00 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c9274de5e19f4537bec7af710c4f8c00 2024-11-15T04:42:04,263 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c9274de5e19f4537bec7af710c4f8c00, entries=150, sequenceid=332, filesize=12.0 K 2024-11-15T04:42:04,264 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c079ffac046c5f34a718ffcef10b362b in 1279ms, sequenceid=332, compaction requested=true 2024-11-15T04:42:04,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/981d7de07cd145cebeaf188b05a4d9a1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/83a1a6e92bdb481584a35dacc778b688, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/29056fd11b414ca882bcaaf53f30554e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/320bca2ad112486c8d2d4381394f7f2e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/6cb3b5170f53428da7a1455156e1373a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/f001eb0909ae42b88096bf7f39f98cf6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0aa824e66c1a4084a91e39f9400ce65b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/1fe071405bba47fa9314ea376a51bb45, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5534053f3a9748369f49c4bd1c057768, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5a6b4c9c3b6945b58729f70c0638391f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/7926ae7b9012426a9416395e37c030f3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/be7679febdad4e13b59e56cfd66b632a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/4375ebf39029483eaa46a0a9a1986f23, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/8d973323ff6e40498cdc448b0871a616, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/c848939e02df4a15ad4c165d64cbb725, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/2126781c46aa4b76a225e9de7c411019, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/b5c05537b9fb438bbdfdeedf8a5e785d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/ecfc6714c434490dbec6b82f789b0ea5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/3c911b1d9d644197980b999900ac8b32, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0ce253eee9384e32afc4cda86a5dce88, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/d7e3b590ddd14aa0bbd89d513927bcd8] to archive 2024-11-15T04:42:04,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:42:04,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/981d7de07cd145cebeaf188b05a4d9a1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/981d7de07cd145cebeaf188b05a4d9a1 2024-11-15T04:42:04,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/83a1a6e92bdb481584a35dacc778b688 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/83a1a6e92bdb481584a35dacc778b688 2024-11-15T04:42:04,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/29056fd11b414ca882bcaaf53f30554e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/29056fd11b414ca882bcaaf53f30554e 2024-11-15T04:42:04,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/320bca2ad112486c8d2d4381394f7f2e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/320bca2ad112486c8d2d4381394f7f2e 2024-11-15T04:42:04,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/6cb3b5170f53428da7a1455156e1373a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/6cb3b5170f53428da7a1455156e1373a 2024-11-15T04:42:04,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/f001eb0909ae42b88096bf7f39f98cf6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/f001eb0909ae42b88096bf7f39f98cf6 2024-11-15T04:42:04,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0aa824e66c1a4084a91e39f9400ce65b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0aa824e66c1a4084a91e39f9400ce65b 2024-11-15T04:42:04,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/1fe071405bba47fa9314ea376a51bb45 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/1fe071405bba47fa9314ea376a51bb45 2024-11-15T04:42:04,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5534053f3a9748369f49c4bd1c057768 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5534053f3a9748369f49c4bd1c057768 2024-11-15T04:42:04,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5a6b4c9c3b6945b58729f70c0638391f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/5a6b4c9c3b6945b58729f70c0638391f 2024-11-15T04:42:04,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/7926ae7b9012426a9416395e37c030f3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/7926ae7b9012426a9416395e37c030f3 2024-11-15T04:42:04,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/be7679febdad4e13b59e56cfd66b632a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/be7679febdad4e13b59e56cfd66b632a 2024-11-15T04:42:04,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/4375ebf39029483eaa46a0a9a1986f23 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/4375ebf39029483eaa46a0a9a1986f23 2024-11-15T04:42:04,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/8d973323ff6e40498cdc448b0871a616 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/8d973323ff6e40498cdc448b0871a616 2024-11-15T04:42:04,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/c848939e02df4a15ad4c165d64cbb725 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/c848939e02df4a15ad4c165d64cbb725 2024-11-15T04:42:04,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/2126781c46aa4b76a225e9de7c411019 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/2126781c46aa4b76a225e9de7c411019 2024-11-15T04:42:04,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/b5c05537b9fb438bbdfdeedf8a5e785d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/b5c05537b9fb438bbdfdeedf8a5e785d 2024-11-15T04:42:04,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/ecfc6714c434490dbec6b82f789b0ea5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/ecfc6714c434490dbec6b82f789b0ea5 2024-11-15T04:42:04,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/3c911b1d9d644197980b999900ac8b32 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/3c911b1d9d644197980b999900ac8b32 2024-11-15T04:42:04,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0ce253eee9384e32afc4cda86a5dce88 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/0ce253eee9384e32afc4cda86a5dce88 2024-11-15T04:42:04,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/d7e3b590ddd14aa0bbd89d513927bcd8 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/d7e3b590ddd14aa0bbd89d513927bcd8 2024-11-15T04:42:04,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/5e43d962cc9a4c09a5c48becf4fca97f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/39875920814b49c5abc259fa7b3bb6df, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9aca64d41ad9402983440013e6c251ac, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/baf735d553ad4df7a32736aecbc59f15, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/61601eb3f0104e38ae98ce3f80b54cca, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2aab2d1b2ce94a64a584a67a7042d5eb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/dbec1025489045fc9a6ff665563258a9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/31fe7ac5d16d4053af4cabde9fb0c980, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/0da6ab77efc24ecfbb1b7c75a6981a61, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/b4df328765954726a7d7717bd0f814b2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/1b9fb4a2ec9c46bbb7491528e4658e25, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/fe232f01ca2243e6be17e896002d4069, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9c7d5746e636401ea8d82aa77f475ea1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/66529fb526114b06a6c934b7b2a702c6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2ec005217873417d800d44e3299247b6, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/d1481b9f0c3449eea775eaa3562f306f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/65d608891c6049549feca2d030e57ebb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/64550e1bf0364d66a3fac7d583097151, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/25a2058865024606aed2f23106e529c8, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/e43978385dcb4384b995c15efc2be4f2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/8e085674cb4944d392fb1ac396d40b63] to archive 2024-11-15T04:42:04,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:42:04,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/5e43d962cc9a4c09a5c48becf4fca97f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/5e43d962cc9a4c09a5c48becf4fca97f 2024-11-15T04:42:04,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/39875920814b49c5abc259fa7b3bb6df to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/39875920814b49c5abc259fa7b3bb6df 2024-11-15T04:42:04,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9aca64d41ad9402983440013e6c251ac to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9aca64d41ad9402983440013e6c251ac 2024-11-15T04:42:04,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/baf735d553ad4df7a32736aecbc59f15 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/baf735d553ad4df7a32736aecbc59f15 2024-11-15T04:42:04,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/61601eb3f0104e38ae98ce3f80b54cca to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/61601eb3f0104e38ae98ce3f80b54cca 2024-11-15T04:42:04,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2aab2d1b2ce94a64a584a67a7042d5eb to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2aab2d1b2ce94a64a584a67a7042d5eb 2024-11-15T04:42:04,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/dbec1025489045fc9a6ff665563258a9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/dbec1025489045fc9a6ff665563258a9 2024-11-15T04:42:04,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/31fe7ac5d16d4053af4cabde9fb0c980 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/31fe7ac5d16d4053af4cabde9fb0c980 2024-11-15T04:42:04,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/0da6ab77efc24ecfbb1b7c75a6981a61 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/0da6ab77efc24ecfbb1b7c75a6981a61 2024-11-15T04:42:04,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/b4df328765954726a7d7717bd0f814b2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/b4df328765954726a7d7717bd0f814b2 2024-11-15T04:42:04,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/1b9fb4a2ec9c46bbb7491528e4658e25 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/1b9fb4a2ec9c46bbb7491528e4658e25 2024-11-15T04:42:04,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/fe232f01ca2243e6be17e896002d4069 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/fe232f01ca2243e6be17e896002d4069 2024-11-15T04:42:04,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9c7d5746e636401ea8d82aa77f475ea1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/9c7d5746e636401ea8d82aa77f475ea1 2024-11-15T04:42:04,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/66529fb526114b06a6c934b7b2a702c6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/66529fb526114b06a6c934b7b2a702c6 2024-11-15T04:42:04,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2ec005217873417d800d44e3299247b6 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/2ec005217873417d800d44e3299247b6 2024-11-15T04:42:04,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/d1481b9f0c3449eea775eaa3562f306f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/d1481b9f0c3449eea775eaa3562f306f 2024-11-15T04:42:04,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/65d608891c6049549feca2d030e57ebb to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/65d608891c6049549feca2d030e57ebb 2024-11-15T04:42:04,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/64550e1bf0364d66a3fac7d583097151 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/64550e1bf0364d66a3fac7d583097151 2024-11-15T04:42:04,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/25a2058865024606aed2f23106e529c8 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/25a2058865024606aed2f23106e529c8 2024-11-15T04:42:04,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/e43978385dcb4384b995c15efc2be4f2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/e43978385dcb4384b995c15efc2be4f2 2024-11-15T04:42:04,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/8e085674cb4944d392fb1ac396d40b63 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/8e085674cb4944d392fb1ac396d40b63 2024-11-15T04:42:04,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/0fe657d52a834fd4964b983cd07c75e8, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/bdb6fb122ba04e049474683431d1a2e4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/e948e1c0d13a45b99a6240a4d7391dc0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c3fbb49656024ae2b0f2d8f23abda4dc, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/fd019617b3534df5aacfde665b55c6d5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/867238d5a754479b86e7640befb70dd2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/923203ce2a8a490f96bc46f27e9f0e13, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d705e86be92946afb4cf99ed933640c9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/5507fe45fe2f4911a67663bcd0663301, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/f7515308172541ef9aa98999ba059e38, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/4f00091a7f9c43ee88533078348baff2, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d80f21df915e4079ac12b5d5ab33467e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/b66efea2a1534fe88c6600c8600b5dd4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/316bc90230084bfdadbf04e71525e26a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/18cf58dc3ea245aeb59793914dc02e4b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7227fdc8d8f4449e92275cbf8a676aad, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a4019f26458b4147ad2522e72ecbdd4c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c3160b645c4849c78f84d577b815b45e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a56621255ed747569dd015bdf31cb4c0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7c0ecb14db404a9fa69c6348946d95ca, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/cbe3808df9ab4308a6a4ef0ef84c2280] to archive 2024-11-15T04:42:04,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:42:04,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/0fe657d52a834fd4964b983cd07c75e8 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/0fe657d52a834fd4964b983cd07c75e8 2024-11-15T04:42:04,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/bdb6fb122ba04e049474683431d1a2e4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/bdb6fb122ba04e049474683431d1a2e4 2024-11-15T04:42:04,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/e948e1c0d13a45b99a6240a4d7391dc0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/e948e1c0d13a45b99a6240a4d7391dc0 2024-11-15T04:42:04,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c3fbb49656024ae2b0f2d8f23abda4dc to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c3fbb49656024ae2b0f2d8f23abda4dc 2024-11-15T04:42:04,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/fd019617b3534df5aacfde665b55c6d5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/fd019617b3534df5aacfde665b55c6d5 2024-11-15T04:42:04,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/867238d5a754479b86e7640befb70dd2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/867238d5a754479b86e7640befb70dd2 2024-11-15T04:42:04,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/923203ce2a8a490f96bc46f27e9f0e13 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/923203ce2a8a490f96bc46f27e9f0e13 2024-11-15T04:42:04,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d705e86be92946afb4cf99ed933640c9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d705e86be92946afb4cf99ed933640c9 2024-11-15T04:42:04,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/5507fe45fe2f4911a67663bcd0663301 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/5507fe45fe2f4911a67663bcd0663301 2024-11-15T04:42:04,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/f7515308172541ef9aa98999ba059e38 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/f7515308172541ef9aa98999ba059e38 2024-11-15T04:42:04,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/4f00091a7f9c43ee88533078348baff2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/4f00091a7f9c43ee88533078348baff2 2024-11-15T04:42:04,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d80f21df915e4079ac12b5d5ab33467e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/d80f21df915e4079ac12b5d5ab33467e 2024-11-15T04:42:04,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/b66efea2a1534fe88c6600c8600b5dd4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/b66efea2a1534fe88c6600c8600b5dd4 2024-11-15T04:42:04,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/316bc90230084bfdadbf04e71525e26a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/316bc90230084bfdadbf04e71525e26a 2024-11-15T04:42:04,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/18cf58dc3ea245aeb59793914dc02e4b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/18cf58dc3ea245aeb59793914dc02e4b 2024-11-15T04:42:04,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7227fdc8d8f4449e92275cbf8a676aad to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7227fdc8d8f4449e92275cbf8a676aad 2024-11-15T04:42:04,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a4019f26458b4147ad2522e72ecbdd4c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a4019f26458b4147ad2522e72ecbdd4c 2024-11-15T04:42:04,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c3160b645c4849c78f84d577b815b45e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c3160b645c4849c78f84d577b815b45e 2024-11-15T04:42:04,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a56621255ed747569dd015bdf31cb4c0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/a56621255ed747569dd015bdf31cb4c0 2024-11-15T04:42:04,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7c0ecb14db404a9fa69c6348946d95ca to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/7c0ecb14db404a9fa69c6348946d95ca 2024-11-15T04:42:04,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/cbe3808df9ab4308a6a4ef0ef84c2280 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/cbe3808df9ab4308a6a4ef0ef84c2280 2024-11-15T04:42:04,365 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/recovered.edits/335.seqid, newMaxSeqId=335, maxSeqId=1 2024-11-15T04:42:04,366 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. 2024-11-15T04:42:04,366 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for c079ffac046c5f34a718ffcef10b362b: Waiting for close lock at 1731645722985Running coprocessor pre-close hooks at 1731645722985Disabling compacts and flushes for region at 1731645722985Disabling writes for close at 1731645722985Obtaining lock to block concurrent updates at 1731645722986 (+1 ms)Preparing flush snapshotting stores in c079ffac046c5f34a718ffcef10b362b at 1731645722986Finished memstore snapshotting TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731645722986Flushing stores of TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b. at 1731645722987 (+1 ms)Flushing c079ffac046c5f34a718ffcef10b362b/A: creating writer at 1731645722987Flushing c079ffac046c5f34a718ffcef10b362b/A: appending metadata at 1731645722993 (+6 ms)Flushing c079ffac046c5f34a718ffcef10b362b/A: closing flushed file at 1731645722993Flushing c079ffac046c5f34a718ffcef10b362b/B: creating writer at 1731645723409 (+416 ms)Flushing c079ffac046c5f34a718ffcef10b362b/B: appending metadata at 1731645723415 (+6 ms)Flushing c079ffac046c5f34a718ffcef10b362b/B: closing flushed file at 1731645723415Flushing c079ffac046c5f34a718ffcef10b362b/C: creating writer at 1731645723828 (+413 ms)Flushing c079ffac046c5f34a718ffcef10b362b/C: appending metadata at 1731645723831 (+3 ms)Flushing c079ffac046c5f34a718ffcef10b362b/C: closing flushed file at 1731645723831Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35720536: reopening flushed file at 1731645724245 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ac7851f: reopening flushed file at 1731645724253 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47940da3: reopening flushed file at 1731645724258 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c079ffac046c5f34a718ffcef10b362b in 1279ms, sequenceid=332, compaction requested=true at 1731645724264 (+6 ms)Writing region close event to WAL at 1731645724361 (+97 ms)Running coprocessor post-close hooks at 1731645724366 (+5 ms)Closed at 1731645724366 2024-11-15T04:42:04,368 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:42:04,369 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=c079ffac046c5f34a718ffcef10b362b, regionState=CLOSED 2024-11-15T04:42:04,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:42:04,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-15T04:42:04,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure c079ffac046c5f34a718ffcef10b362b, server=26747e6ada4f,42935,1731645581086 in 1.5420 sec 2024-11-15T04:42:04,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-15T04:42:04,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c079ffac046c5f34a718ffcef10b362b, UNASSIGN in 1.5480 sec 2024-11-15T04:42:04,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-15T04:42:04,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5530 sec 2024-11-15T04:42:04,383 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645724383"}]},"ts":"1731645724383"} 2024-11-15T04:42:04,385 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T04:42:04,385 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T04:42:04,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5670 sec 2024-11-15T04:42:04,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T04:42:04,984 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:04,985 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T04:42:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:04,989 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T04:42:04,991 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:04,995 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:42:04,997 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/recovered.edits] 2024-11-15T04:42:05,000 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/37ff44e5cc6b46e68a2e893841fa0432 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/37ff44e5cc6b46e68a2e893841fa0432 2024-11-15T04:42:05,001 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/85ae244fa49b4b45be2f64dfa9ebd8ab to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/85ae244fa49b4b45be2f64dfa9ebd8ab 2024-11-15T04:42:05,003 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/a561a993734142e88827a84a8829f252 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/A/a561a993734142e88827a84a8829f252 2024-11-15T04:42:05,005 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/3d92fe0565f3469b9b09af97105a0607 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/3d92fe0565f3469b9b09af97105a0607 2024-11-15T04:42:05,006 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/4d7cad15a89f4b468b1cbf992a64a3a9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/4d7cad15a89f4b468b1cbf992a64a3a9 2024-11-15T04:42:05,007 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/6b662f48e437497bb46dfadfc693c6d2 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/B/6b662f48e437497bb46dfadfc693c6d2 2024-11-15T04:42:05,010 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/82dab27b538b4227818164285e4e9770 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/82dab27b538b4227818164285e4e9770 2024-11-15T04:42:05,011 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/b30d3f12cb71487eab4b5b612587fec8 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/b30d3f12cb71487eab4b5b612587fec8 2024-11-15T04:42:05,012 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c9274de5e19f4537bec7af710c4f8c00 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/C/c9274de5e19f4537bec7af710c4f8c00 2024-11-15T04:42:05,016 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/recovered.edits/335.seqid to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b/recovered.edits/335.seqid 2024-11-15T04:42:05,016 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/c079ffac046c5f34a718ffcef10b362b 2024-11-15T04:42:05,016 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T04:42:05,019 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:05,021 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T04:42:05,023 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T04:42:05,024 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:05,024 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T04:42:05,024 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731645725024"}]},"ts":"9223372036854775807"} 2024-11-15T04:42:05,026 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T04:42:05,026 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c079ffac046c5f34a718ffcef10b362b, NAME => 'TestAcidGuarantees,,1731645697918.c079ffac046c5f34a718ffcef10b362b.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T04:42:05,026 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T04:42:05,026 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731645725026"}]},"ts":"9223372036854775807"} 2024-11-15T04:42:05,028 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T04:42:05,028 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:05,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-15T04:42:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T04:42:05,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T04:42:05,101 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:05,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 240), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 443), ProcessCount=11 (was 11), AvailableMemoryMB=14631 (was 14668) 2024-11-15T04:42:05,121 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=14631 2024-11-15T04:42:05,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T04:42:05,123 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T04:42:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:05,125 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T04:42:05,125 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:05,125 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-15T04:42:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T04:42:05,126 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T04:42:05,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742351_1527 (size=959) 2024-11-15T04:42:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T04:42:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T04:42:05,538 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6 2024-11-15T04:42:05,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742352_1528 (size=53) 2024-11-15T04:42:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T04:42:05,950 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:42:05,950 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1f3526eb9bd778c7571f046690073138, disabling compactions & flushes 2024-11-15T04:42:05,950 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:05,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:05,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. after waiting 0 ms 2024-11-15T04:42:05,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:05,951 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:05,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1f3526eb9bd778c7571f046690073138: Waiting for close lock at 1731645725950Disabling compacts and flushes for region at 1731645725950Disabling writes for close at 1731645725951 (+1 ms)Writing region close event to WAL at 1731645725951Closed at 1731645725951 2024-11-15T04:42:05,953 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T04:42:05,954 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731645725953"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731645725953"}]},"ts":"1731645725953"} 2024-11-15T04:42:05,956 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T04:42:05,957 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T04:42:05,958 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645725957"}]},"ts":"1731645725957"} 2024-11-15T04:42:05,960 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T04:42:05,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, ASSIGN}] 2024-11-15T04:42:05,962 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, ASSIGN 2024-11-15T04:42:05,962 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, ASSIGN; state=OFFLINE, location=26747e6ada4f,42935,1731645581086; forceNewPlan=false, retain=false 2024-11-15T04:42:06,114 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1f3526eb9bd778c7571f046690073138, regionState=OPENING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:06,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, ASSIGN because future has completed 2024-11-15T04:42:06,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:42:06,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T04:42:06,276 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:06,276 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:42:06,277 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,277 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:42:06,277 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,277 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,280 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,282 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f3526eb9bd778c7571f046690073138 columnFamilyName A 2024-11-15T04:42:06,282 DEBUG [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:06,283 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(327): Store=1f3526eb9bd778c7571f046690073138/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:42:06,283 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,284 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f3526eb9bd778c7571f046690073138 columnFamilyName B 2024-11-15T04:42:06,284 DEBUG [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:06,284 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(327): Store=1f3526eb9bd778c7571f046690073138/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:42:06,284 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,285 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f3526eb9bd778c7571f046690073138 columnFamilyName C 2024-11-15T04:42:06,285 DEBUG [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:06,285 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(327): Store=1f3526eb9bd778c7571f046690073138/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:42:06,285 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,286 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,286 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,287 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,287 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,287 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:42:06,288 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,289 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T04:42:06,289 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 1f3526eb9bd778c7571f046690073138; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66862416, jitterRate=-0.003672361373901367}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:42:06,289 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:06,290 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 1f3526eb9bd778c7571f046690073138: Running coprocessor pre-open hook at 1731645726278Writing region info on filesystem at 1731645726278Initializing all the Stores at 1731645726279 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645726279Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645726279Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645726279Cleaning up temporary data from old regions at 1731645726287 (+8 ms)Running coprocessor post-open hooks at 1731645726290 (+3 ms)Region opened successfully at 1731645726290 2024-11-15T04:42:06,291 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., pid=140, masterSystemTime=1731645726270 2024-11-15T04:42:06,292 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:06,292 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:06,292 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=1f3526eb9bd778c7571f046690073138, regionState=OPEN, openSeqNum=2, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:06,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:42:06,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-15T04:42:06,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 in 176 msec 2024-11-15T04:42:06,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-15T04:42:06,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, ASSIGN in 336 msec 2024-11-15T04:42:06,298 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T04:42:06,298 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645726298"}]},"ts":"1731645726298"} 2024-11-15T04:42:06,299 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T04:42:06,300 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T04:42:06,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1770 sec 2024-11-15T04:42:07,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-15T04:42:07,274 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:07,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b0d3b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,279 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb9fe43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,283 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62518d6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,284 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,287 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T04:42:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 26747e6ada4f,42655,1731645580243 2024-11-15T04:42:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1acd3992 2024-11-15T04:42:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T04:42:07,288 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T04:42:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T04:42:07,290 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T04:42:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:07,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742353_1529 (size=995) 2024-11-15T04:42:07,302 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T04:42:07,302 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T04:42:07,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T04:42:07,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, REOPEN/MOVE}] 2024-11-15T04:42:07,308 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, REOPEN/MOVE 2024-11-15T04:42:07,309 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1f3526eb9bd778c7571f046690073138, regionState=CLOSING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:07,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, REOPEN/MOVE because future has completed 2024-11-15T04:42:07,311 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T04:42:07,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:42:07,464 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,464 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T04:42:07,465 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 1f3526eb9bd778c7571f046690073138, disabling compactions & flushes 2024-11-15T04:42:07,465 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:07,465 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:07,465 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. after waiting 0 ms 2024-11-15T04:42:07,465 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:07,471 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T04:42:07,472 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:07,473 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 1f3526eb9bd778c7571f046690073138: Waiting for close lock at 1731645727465Running coprocessor pre-close hooks at 1731645727465Disabling compacts and flushes for region at 1731645727465Disabling writes for close at 1731645727465Writing region close event to WAL at 1731645727466 (+1 ms)Running coprocessor post-close hooks at 1731645727472 (+6 ms)Closed at 1731645727472 2024-11-15T04:42:07,473 WARN [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 1f3526eb9bd778c7571f046690073138 to self. 2024-11-15T04:42:07,475 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,476 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1f3526eb9bd778c7571f046690073138, regionState=CLOSED 2024-11-15T04:42:07,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:42:07,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-15T04:42:07,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 in 167 msec 2024-11-15T04:42:07,480 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, REOPEN/MOVE; state=CLOSED, location=26747e6ada4f,42935,1731645581086; forceNewPlan=false, retain=true 2024-11-15T04:42:07,631 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1f3526eb9bd778c7571f046690073138, regionState=OPENING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:07,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, REOPEN/MOVE because future has completed 2024-11-15T04:42:07,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:42:07,793 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:07,793 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} 2024-11-15T04:42:07,793 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,793 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T04:42:07,794 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,794 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,796 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,798 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f3526eb9bd778c7571f046690073138 columnFamilyName A 2024-11-15T04:42:07,800 DEBUG [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:07,801 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(327): Store=1f3526eb9bd778c7571f046690073138/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:42:07,802 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,803 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f3526eb9bd778c7571f046690073138 columnFamilyName B 2024-11-15T04:42:07,803 DEBUG [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:07,804 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(327): Store=1f3526eb9bd778c7571f046690073138/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:42:07,804 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,805 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f3526eb9bd778c7571f046690073138 columnFamilyName C 2024-11-15T04:42:07,806 DEBUG [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:07,806 INFO [StoreOpener-1f3526eb9bd778c7571f046690073138-1 {}] regionserver.HStore(327): Store=1f3526eb9bd778c7571f046690073138/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T04:42:07,806 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,807 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,808 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,809 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,809 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,809 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T04:42:07,810 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,811 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 1f3526eb9bd778c7571f046690073138; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74435309, jitterRate=0.10917253792285919}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T04:42:07,811 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,812 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 1f3526eb9bd778c7571f046690073138: Running coprocessor pre-open hook at 1731645727794Writing region info on filesystem at 1731645727794Initializing all the Stores at 1731645727795 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645727796 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645727796Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731645727796Cleaning up temporary data from old regions at 1731645727809 (+13 ms)Running coprocessor post-open hooks at 1731645727811 (+2 ms)Region opened successfully at 1731645727811 2024-11-15T04:42:07,812 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., pid=145, masterSystemTime=1731645727788 2024-11-15T04:42:07,814 DEBUG [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:07,814 INFO [RS_OPEN_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:07,814 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=1f3526eb9bd778c7571f046690073138, regionState=OPEN, openSeqNum=5, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:07,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:42:07,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-15T04:42:07,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 in 182 msec 2024-11-15T04:42:07,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-15T04:42:07,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, REOPEN/MOVE in 510 msec 2024-11-15T04:42:07,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-15T04:42:07,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-11-15T04:42:07,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 531 msec 2024-11-15T04:42:07,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-15T04:42:07,823 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:07,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af38ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6317cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ec0745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eb37c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,827 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1466031b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@369b956b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d415d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,829 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67edc71b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,831 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbb4da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e839bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,832 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,832 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4f590e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2258c8f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31b8df26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,835 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,835 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32150b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,837 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0bf235, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50101ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,838 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c11e3fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,840 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb7505d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7631c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,841 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,841 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a730da1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,843 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@782be2ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b8ab27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,845 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4a632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,847 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1442c3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e1056f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,849 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77332494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,850 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e6b3f83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@294719e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26747e6ada4f,42655,-1 for getting cluster id 2024-11-15T04:42:07,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T04:42:07,851 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fb97bd75-b06f-46e2-9d7c-0f171115d342' 2024-11-15T04:42:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T04:42:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fb97bd75-b06f-46e2-9d7c-0f171115d342" 2024-11-15T04:42:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a76d93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26747e6ada4f,42655,-1] 2024-11-15T04:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T04:42:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:07,852 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T04:42:07,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ad06b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T04:42:07,853 DEBUG [Thread-2377 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,853 DEBUG [Thread-2371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,853 DEBUG [Thread-2366 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,854 DEBUG [Thread-2358 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,854 DEBUG [Thread-2360 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,854 DEBUG [Thread-2362 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,854 DEBUG [Thread-2375 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,854 DEBUG [Thread-2369 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,854 DEBUG [Thread-2373 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,854 DEBUG [Thread-2364 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T04:42:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:42:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,856 DEBUG [Thread-2362 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26747e6ada4f,42935,1731645581086, seqNum=-1] 2024-11-15T04:42:07,856 DEBUG [Thread-2362 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T04:42:07,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38828, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:42:07,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:42:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-15T04:42:07,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:42:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T04:42:07,858 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:42:07,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T04:42:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:42:07,859 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:42:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:42:07,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:42:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:42:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:42:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:42:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:42:07,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5] 2024-11-15T04:42:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,865 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:07,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:42:07,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115917e279bcc1e4756853ead987554afbb_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645727863/Put/seqid=0 2024-11-15T04:42:07,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742354_1530 (size=12154) 2024-11-15T04:42:07,895 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:07,899 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115917e279bcc1e4756853ead987554afbb_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115917e279bcc1e4756853ead987554afbb_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:07,900 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/b9dee4a50e414b10ba9785736736ddbf, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:07,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/b9dee4a50e414b10ba9785736736ddbf is 175, key is test_row_0/A:col10/1731645727863/Put/seqid=0 2024-11-15T04:42:07,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742355_1531 (size=30955) 2024-11-15T04:42:07,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645787902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:07,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645787903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:07,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645787905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:07,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:07,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645787906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:07,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:07,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:07,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:07,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645787907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:07,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:07,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:07,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:07,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:07,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T04:42:08,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T04:42:08,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:08,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645788012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645788013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645788013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645788013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645788013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,015 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T04:42:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T04:42:08,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645788222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645788223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645788223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645788223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645788223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,305 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/b9dee4a50e414b10ba9785736736ddbf 2024-11-15T04:42:08,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T04:42:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/7875c96693614b67a2901779659cf320 is 50, key is test_row_0/B:col10/1731645727863/Put/seqid=0 2024-11-15T04:42:08,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742356_1532 (size=12001) 2024-11-15T04:42:08,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T04:42:08,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T04:42:08,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645788532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645788533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645788543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645788543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645788543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:08,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:08,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T04:42:08,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:08,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/7875c96693614b67a2901779659cf320 2024-11-15T04:42:08,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/49a3f134279a4967aaacf23cb4c95eea is 50, key is test_row_0/C:col10/1731645727863/Put/seqid=0 2024-11-15T04:42:08,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742357_1533 (size=12001) 2024-11-15T04:42:08,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T04:42:08,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T04:42:08,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:08,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:08,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T04:42:09,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645789043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:09,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645789043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:09,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645789052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:09,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645789052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:09,054 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:09,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645789052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:09,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:09,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:09,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:09,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T04:42:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:09,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:09,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:09,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/49a3f134279a4967aaacf23cb4c95eea 2024-11-15T04:42:09,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/b9dee4a50e414b10ba9785736736ddbf as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b9dee4a50e414b10ba9785736736ddbf 2024-11-15T04:42:09,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b9dee4a50e414b10ba9785736736ddbf, entries=150, sequenceid=15, filesize=30.2 K 2024-11-15T04:42:09,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/7875c96693614b67a2901779659cf320 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7875c96693614b67a2901779659cf320 2024-11-15T04:42:09,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7875c96693614b67a2901779659cf320, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T04:42:09,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/49a3f134279a4967aaacf23cb4c95eea as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/49a3f134279a4967aaacf23cb4c95eea 2024-11-15T04:42:09,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/49a3f134279a4967aaacf23cb4c95eea, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T04:42:09,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1f3526eb9bd778c7571f046690073138 in 1310ms, sequenceid=15, compaction requested=false 2024-11-15T04:42:09,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:09,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-15T04:42:09,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:09,235 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T04:42:09,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ef59ae8dac514258b06692eb09c7beee_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645727900/Put/seqid=0 2024-11-15T04:42:09,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742358_1534 (size=12154) 2024-11-15T04:42:09,361 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T04:42:09,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:09,651 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ef59ae8dac514258b06692eb09c7beee_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ef59ae8dac514258b06692eb09c7beee_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:09,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/a6d7d7435a10401aa3f654fe31f7614d, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:09,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/a6d7d7435a10401aa3f654fe31f7614d is 175, key is test_row_0/A:col10/1731645727900/Put/seqid=0 2024-11-15T04:42:09,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742359_1535 (size=30955) 2024-11-15T04:42:09,882 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T04:42:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T04:42:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:10,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:10,057 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/a6d7d7435a10401aa3f654fe31f7614d 2024-11-15T04:42:10,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/77f126ed89c5449f8e1233db88274c4f is 50, key is test_row_0/B:col10/1731645727900/Put/seqid=0 2024-11-15T04:42:10,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742360_1536 (size=12001) 2024-11-15T04:42:10,067 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/77f126ed89c5449f8e1233db88274c4f 2024-11-15T04:42:10,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645790066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645790067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645790068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,072 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/1a5a6dd4b8794a69a69cbc4f6330021d is 50, key is test_row_0/C:col10/1731645727900/Put/seqid=0 2024-11-15T04:42:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645790073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645790073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,081 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:10,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,081 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,081 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742361_1537 (size=12001) 2024-11-15T04:42:10,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645790173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645790173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,176 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645790182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645790383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645790383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645790392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,489 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/1a5a6dd4b8794a69a69cbc4f6330021d 2024-11-15T04:42:10,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/a6d7d7435a10401aa3f654fe31f7614d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/a6d7d7435a10401aa3f654fe31f7614d 2024-11-15T04:42:10,495 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/a6d7d7435a10401aa3f654fe31f7614d, entries=150, sequenceid=41, filesize=30.2 K 2024-11-15T04:42:10,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/77f126ed89c5449f8e1233db88274c4f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77f126ed89c5449f8e1233db88274c4f 2024-11-15T04:42:10,498 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77f126ed89c5449f8e1233db88274c4f, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T04:42:10,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/1a5a6dd4b8794a69a69cbc4f6330021d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/1a5a6dd4b8794a69a69cbc4f6330021d 2024-11-15T04:42:10,502 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/1a5a6dd4b8794a69a69cbc4f6330021d, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T04:42:10,507 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1f3526eb9bd778c7571f046690073138 in 1272ms, sequenceid=41, compaction requested=false 2024-11-15T04:42:10,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:10,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:10,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-15T04:42:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-15T04:42:10,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-15T04:42:10,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6490 sec 2024-11-15T04:42:10,511 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.6540 sec 2024-11-15T04:42:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:10,694 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:10,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:42:10,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111593017244cdec449d8dbaa23884621a14_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645730693/Put/seqid=0 2024-11-15T04:42:10,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742362_1538 (size=12154) 2024-11-15T04:42:10,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645790724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645790725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645790728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645790833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645790833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:10,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645790842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:10,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:10,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:11,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645791043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645791043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645791053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:11,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:11,106 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111593017244cdec449d8dbaa23884621a14_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111593017244cdec449d8dbaa23884621a14_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:11,107 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/23dc851fd0244199bd5d626321c48769, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:11,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/23dc851fd0244199bd5d626321c48769 is 175, key is test_row_0/A:col10/1731645730693/Put/seqid=0 2024-11-15T04:42:11,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742363_1539 (size=30955) 2024-11-15T04:42:11,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645791352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645791352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:11,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:11,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645791362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:11,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/23dc851fd0244199bd5d626321c48769 2024-11-15T04:42:11,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/3436e5da23e74f6fbc35899244df9a3d is 50, key is test_row_0/B:col10/1731645730693/Put/seqid=0 2024-11-15T04:42:11,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742364_1540 (size=12001) 2024-11-15T04:42:11,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645791863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:11,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645791863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:11,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:11,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645791872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:11,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:11,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:11,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/3436e5da23e74f6fbc35899244df9a3d 2024-11-15T04:42:11,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/2b9dd1be0ec34f48abc93f9f48b8c8f9 is 50, key is test_row_0/C:col10/1731645730693/Put/seqid=0 2024-11-15T04:42:11,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742365_1541 (size=12001) 2024-11-15T04:42:11,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/2b9dd1be0ec34f48abc93f9f48b8c8f9 2024-11-15T04:42:11,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/23dc851fd0244199bd5d626321c48769 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/23dc851fd0244199bd5d626321c48769 2024-11-15T04:42:11,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/23dc851fd0244199bd5d626321c48769, entries=150, sequenceid=52, filesize=30.2 K 2024-11-15T04:42:11,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/3436e5da23e74f6fbc35899244df9a3d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/3436e5da23e74f6fbc35899244df9a3d 2024-11-15T04:42:11,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/3436e5da23e74f6fbc35899244df9a3d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T04:42:11,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/2b9dd1be0ec34f48abc93f9f48b8c8f9 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/2b9dd1be0ec34f48abc93f9f48b8c8f9 2024-11-15T04:42:11,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/2b9dd1be0ec34f48abc93f9f48b8c8f9, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T04:42:11,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1f3526eb9bd778c7571f046690073138 in 1252ms, sequenceid=52, compaction requested=true 2024-11-15T04:42:11,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:11,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:42:11,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:11,947 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:42:11,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:42:11,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:11,947 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:42:11,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:42:11,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:11,947 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:42:11,947 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:42:11,947 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/A is initiating minor compaction (all files) 2024-11-15T04:42:11,947 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/B is initiating minor compaction (all files) 2024-11-15T04:42:11,948 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/A in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:11,948 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/B in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:11,948 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7875c96693614b67a2901779659cf320, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77f126ed89c5449f8e1233db88274c4f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/3436e5da23e74f6fbc35899244df9a3d] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=35.2 K 2024-11-15T04:42:11,948 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b9dee4a50e414b10ba9785736736ddbf, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/a6d7d7435a10401aa3f654fe31f7614d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/23dc851fd0244199bd5d626321c48769] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=90.7 K 2024-11-15T04:42:11,948 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:11,948 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b9dee4a50e414b10ba9785736736ddbf, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/a6d7d7435a10401aa3f654fe31f7614d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/23dc851fd0244199bd5d626321c48769] 2024-11-15T04:42:11,948 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 7875c96693614b67a2901779659cf320, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731645727863 2024-11-15T04:42:11,948 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9dee4a50e414b10ba9785736736ddbf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731645727863 2024-11-15T04:42:11,948 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f126ed89c5449f8e1233db88274c4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731645727900 2024-11-15T04:42:11,948 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6d7d7435a10401aa3f654fe31f7614d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731645727900 2024-11-15T04:42:11,949 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 3436e5da23e74f6fbc35899244df9a3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645730063 2024-11-15T04:42:11,949 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23dc851fd0244199bd5d626321c48769, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645730063 2024-11-15T04:42:11,952 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:11,954 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:11,954 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411156ab95f8eebc242b5a8afe7a1c1a1567b_1f3526eb9bd778c7571f046690073138 store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:11,954 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/5c85c038f20e4d719356b6cdbf68f4ff is 50, key is test_row_0/B:col10/1731645730693/Put/seqid=0 2024-11-15T04:42:11,955 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411156ab95f8eebc242b5a8afe7a1c1a1567b_1f3526eb9bd778c7571f046690073138, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:11,955 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156ab95f8eebc242b5a8afe7a1c1a1567b_1f3526eb9bd778c7571f046690073138 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:11,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742366_1542 (size=12104) 2024-11-15T04:42:11,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742367_1543 (size=4469) 2024-11-15T04:42:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-15T04:42:12,023 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:12,023 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:42:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-15T04:42:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T04:42:12,024 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:42:12,025 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:42:12,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:42:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:12,096 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:12,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T04:42:12,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115007d47024769419c878eb20d0d730b17_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645730727/Put/seqid=0 2024-11-15T04:42:12,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742368_1544 (size=14594) 2024-11-15T04:42:12,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645792109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645792110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T04:42:12,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645792222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645792222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:12,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:12,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T04:42:12,367 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/5c85c038f20e4d719356b6cdbf68f4ff as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/5c85c038f20e4d719356b6cdbf68f4ff 2024-11-15T04:42:12,369 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#A#compaction#458 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:12,369 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/c31e79ee672d4147bd923047a8572930 is 175, key is test_row_0/A:col10/1731645730693/Put/seqid=0 2024-11-15T04:42:12,373 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f3526eb9bd778c7571f046690073138/B of 1f3526eb9bd778c7571f046690073138 into 5c85c038f20e4d719356b6cdbf68f4ff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:12,373 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:12,373 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/B, priority=13, startTime=1731645731947; duration=0sec 2024-11-15T04:42:12,373 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:12,373 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:B 2024-11-15T04:42:12,374 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:42:12,374 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:42:12,375 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/C is initiating minor compaction (all files) 2024-11-15T04:42:12,375 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/C in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,375 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/49a3f134279a4967aaacf23cb4c95eea, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/1a5a6dd4b8794a69a69cbc4f6330021d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/2b9dd1be0ec34f48abc93f9f48b8c8f9] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=35.2 K 2024-11-15T04:42:12,375 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a3f134279a4967aaacf23cb4c95eea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731645727863 2024-11-15T04:42:12,375 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a5a6dd4b8794a69a69cbc4f6330021d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731645727900 2024-11-15T04:42:12,375 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9dd1be0ec34f48abc93f9f48b8c8f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645730063 2024-11-15T04:42:12,386 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:12,387 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/539ed8d4b91449baa621359003ed4a89 is 50, key is test_row_0/C:col10/1731645730693/Put/seqid=0 2024-11-15T04:42:12,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742369_1545 (size=31058) 2024-11-15T04:42:12,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742370_1546 (size=12104) 2024-11-15T04:42:12,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645792432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645792432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:12,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:12,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,506 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:12,509 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115007d47024769419c878eb20d0d730b17_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115007d47024769419c878eb20d0d730b17_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:12,510 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/10684892722e4140b4a082a7a9679c4a, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:12,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/10684892722e4140b4a082a7a9679c4a is 175, key is test_row_0/A:col10/1731645730727/Put/seqid=0 2024-11-15T04:42:12,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742371_1547 (size=39549) 2024-11-15T04:42:12,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T04:42:12,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645792743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645792743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,792 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/c31e79ee672d4147bd923047a8572930 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/c31e79ee672d4147bd923047a8572930 2024-11-15T04:42:12,795 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f3526eb9bd778c7571f046690073138/A of 1f3526eb9bd778c7571f046690073138 into c31e79ee672d4147bd923047a8572930(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:12,796 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:12,796 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/A, priority=13, startTime=1731645731947; duration=0sec 2024-11-15T04:42:12,796 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:12,796 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:A 2024-11-15T04:42:12,799 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/539ed8d4b91449baa621359003ed4a89 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/539ed8d4b91449baa621359003ed4a89 2024-11-15T04:42:12,802 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f3526eb9bd778c7571f046690073138/C of 1f3526eb9bd778c7571f046690073138 into 539ed8d4b91449baa621359003ed4a89(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:12,802 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:12,802 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/C, priority=13, startTime=1731645731947; duration=0sec 2024-11-15T04:42:12,803 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:12,803 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:C 2024-11-15T04:42:12,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645792873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645792873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,874 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,874 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645792882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:12,885 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:12,915 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/10684892722e4140b4a082a7a9679c4a 2024-11-15T04:42:12,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/05908fe6edac49a48b92ce58870fa500 is 50, key is test_row_0/B:col10/1731645730727/Put/seqid=0 2024-11-15T04:42:12,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742372_1548 (size=12001) 2024-11-15T04:42:12,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:12,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:12,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:12,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:13,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:13,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T04:42:13,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:13,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:13,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645793253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:13,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645793253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:13,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:13,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/05908fe6edac49a48b92ce58870fa500 2024-11-15T04:42:13,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/057a22b8c39f48c0b78b67066ca67b7e is 50, key is test_row_0/C:col10/1731645730727/Put/seqid=0 2024-11-15T04:42:13,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742373_1549 (size=12001) 2024-11-15T04:42:13,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:13,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:13,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:13,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:13,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:13,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/057a22b8c39f48c0b78b67066ca67b7e 2024-11-15T04:42:13,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/10684892722e4140b4a082a7a9679c4a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/10684892722e4140b4a082a7a9679c4a 2024-11-15T04:42:13,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/10684892722e4140b4a082a7a9679c4a, entries=200, sequenceid=79, filesize=38.6 K 2024-11-15T04:42:13,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/05908fe6edac49a48b92ce58870fa500 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/05908fe6edac49a48b92ce58870fa500 2024-11-15T04:42:13,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/05908fe6edac49a48b92ce58870fa500, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T04:42:13,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/057a22b8c39f48c0b78b67066ca67b7e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/057a22b8c39f48c0b78b67066ca67b7e 2024-11-15T04:42:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/057a22b8c39f48c0b78b67066ca67b7e, entries=150, sequenceid=79, filesize=11.7 K 2024-11-15T04:42:13,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1f3526eb9bd778c7571f046690073138 in 1653ms, sequenceid=79, compaction requested=false 2024-11-15T04:42:13,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-15T04:42:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:13,856 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T04:42:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157596d087f7574604b57ad5cabaadf6c5_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_1/A:col10/1731645732109/Put/seqid=0 2024-11-15T04:42:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742374_1550 (size=9714) 2024-11-15T04:42:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T04:42:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,269 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157596d087f7574604b57ad5cabaadf6c5_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157596d087f7574604b57ad5cabaadf6c5_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/8b5ba5acbde04edb808adc1cfaf32ec0, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/8b5ba5acbde04edb808adc1cfaf32ec0 is 175, key is test_row_1/A:col10/1731645732109/Put/seqid=0 2024-11-15T04:42:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:14,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742375_1551 (size=22361) 2024-11-15T04:42:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,275 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/8b5ba5acbde04edb808adc1cfaf32ec0 2024-11-15T04:42:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/7a861cac5f4a458f9641993a956179ad is 50, key is test_row_1/B:col10/1731645732109/Put/seqid=0 2024-11-15T04:42:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742376_1552 (size=9657) 2024-11-15T04:42:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,290 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/7a861cac5f4a458f9641993a956179ad 2024-11-15T04:42:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/5986e170cba44e5084a455c2ced8ebb4 is 50, key is test_row_1/C:col10/1731645732109/Put/seqid=0 2024-11-15T04:42:14,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742377_1553 (size=9657) 2024-11-15T04:42:14,307 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/5986e170cba44e5084a455c2ced8ebb4 2024-11-15T04:42:14,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/8b5ba5acbde04edb808adc1cfaf32ec0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8b5ba5acbde04edb808adc1cfaf32ec0 2024-11-15T04:42:14,317 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8b5ba5acbde04edb808adc1cfaf32ec0, entries=100, sequenceid=91, filesize=21.8 K 2024-11-15T04:42:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/7a861cac5f4a458f9641993a956179ad as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7a861cac5f4a458f9641993a956179ad 2024-11-15T04:42:14,322 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7a861cac5f4a458f9641993a956179ad, entries=100, sequenceid=91, filesize=9.4 K 2024-11-15T04:42:14,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/5986e170cba44e5084a455c2ced8ebb4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/5986e170cba44e5084a455c2ced8ebb4 2024-11-15T04:42:14,328 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/5986e170cba44e5084a455c2ced8ebb4, entries=100, sequenceid=91, filesize=9.4 K 2024-11-15T04:42:14,329 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=147.60 KB/151140 for 1f3526eb9bd778c7571f046690073138 in 473ms, sequenceid=91, compaction requested=true 2024-11-15T04:42:14,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:14,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:14,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-15T04:42:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-15T04:42:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:14,332 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:14,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T04:42:14,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-15T04:42:14,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3060 sec 2024-11-15T04:42:14,334 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.3100 sec 2024-11-15T04:42:14,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111517dff034f277429389db37c4300a5240_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645734328/Put/seqid=0 2024-11-15T04:42:14,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742378_1554 (size=17034) 2024-11-15T04:42:14,344 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:14,348 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111517dff034f277429389db37c4300a5240_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111517dff034f277429389db37c4300a5240_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:14,349 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/b4dbbdba64044af2a9d113dc5249dc03, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:14,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/b4dbbdba64044af2a9d113dc5249dc03 is 175, key is test_row_0/A:col10/1731645734328/Put/seqid=0 2024-11-15T04:42:14,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645794346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645794347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,351 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,352 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:14,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742379_1555 (size=48139) 2024-11-15T04:42:14,353 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/b4dbbdba64044af2a9d113dc5249dc03 2024-11-15T04:42:14,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/77ae11587400456d817fd435b659b7e1 is 50, key is test_row_0/B:col10/1731645734328/Put/seqid=0 2024-11-15T04:42:14,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742380_1556 (size=12001) 2024-11-15T04:42:14,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645794453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:14,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645794462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:14,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645794663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:14,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645794673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:14,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/77ae11587400456d817fd435b659b7e1 2024-11-15T04:42:14,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/87cee007f2074de88dea7c4cb32c009b is 50, key is test_row_0/C:col10/1731645734328/Put/seqid=0 2024-11-15T04:42:14,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742381_1557 (size=12001) 2024-11-15T04:42:14,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645794893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645794893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,896 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:14,896 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:14,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645794903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,904 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:14,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645794973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645794983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:15,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/87cee007f2074de88dea7c4cb32c009b 2024-11-15T04:42:15,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/b4dbbdba64044af2a9d113dc5249dc03 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b4dbbdba64044af2a9d113dc5249dc03 2024-11-15T04:42:15,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b4dbbdba64044af2a9d113dc5249dc03, entries=250, sequenceid=117, filesize=47.0 K 2024-11-15T04:42:15,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/77ae11587400456d817fd435b659b7e1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77ae11587400456d817fd435b659b7e1 2024-11-15T04:42:15,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77ae11587400456d817fd435b659b7e1, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T04:42:15,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/87cee007f2074de88dea7c4cb32c009b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/87cee007f2074de88dea7c4cb32c009b 2024-11-15T04:42:15,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/87cee007f2074de88dea7c4cb32c009b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T04:42:15,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 1f3526eb9bd778c7571f046690073138 in 856ms, sequenceid=117, compaction requested=true 2024-11-15T04:42:15,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:15,189 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:42:15,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:42:15,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:15,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:42:15,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:15,189 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:42:15,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:42:15,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:15,190 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:42:15,190 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/A is initiating minor compaction (all files) 2024-11-15T04:42:15,190 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:42:15,190 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/A in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:15,190 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/B is initiating minor compaction (all files) 2024-11-15T04:42:15,190 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/B in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:15,190 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/c31e79ee672d4147bd923047a8572930, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/10684892722e4140b4a082a7a9679c4a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8b5ba5acbde04edb808adc1cfaf32ec0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b4dbbdba64044af2a9d113dc5249dc03] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=137.8 K 2024-11-15T04:42:15,190 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/5c85c038f20e4d719356b6cdbf68f4ff, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/05908fe6edac49a48b92ce58870fa500, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7a861cac5f4a458f9641993a956179ad, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77ae11587400456d817fd435b659b7e1] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=44.7 K 2024-11-15T04:42:15,190 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:15,190 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/c31e79ee672d4147bd923047a8572930, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/10684892722e4140b4a082a7a9679c4a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8b5ba5acbde04edb808adc1cfaf32ec0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b4dbbdba64044af2a9d113dc5249dc03] 2024-11-15T04:42:15,191 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c85c038f20e4d719356b6cdbf68f4ff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645730063 2024-11-15T04:42:15,191 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting c31e79ee672d4147bd923047a8572930, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645730063 2024-11-15T04:42:15,191 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 05908fe6edac49a48b92ce58870fa500, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731645730727 2024-11-15T04:42:15,191 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a861cac5f4a458f9641993a956179ad, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731645732102 2024-11-15T04:42:15,191 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10684892722e4140b4a082a7a9679c4a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731645730724 2024-11-15T04:42:15,192 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 77ae11587400456d817fd435b659b7e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645734322 2024-11-15T04:42:15,192 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b5ba5acbde04edb808adc1cfaf32ec0, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731645732102 2024-11-15T04:42:15,192 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4dbbdba64044af2a9d113dc5249dc03, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645734322 2024-11-15T04:42:15,201 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#B#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:15,201 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/4fa310472ee243a6b7521d1a0c8ece0e is 50, key is test_row_0/B:col10/1731645734328/Put/seqid=0 2024-11-15T04:42:15,202 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:15,205 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115b58d1593f56a450ead613836754d7dce_1f3526eb9bd778c7571f046690073138 store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:15,207 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115b58d1593f56a450ead613836754d7dce_1f3526eb9bd778c7571f046690073138, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:15,207 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b58d1593f56a450ead613836754d7dce_1f3526eb9bd778c7571f046690073138 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:15,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742382_1558 (size=12241) 2024-11-15T04:42:15,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742383_1559 (size=4469) 2024-11-15T04:42:15,212 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/4fa310472ee243a6b7521d1a0c8ece0e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/4fa310472ee243a6b7521d1a0c8ece0e 2024-11-15T04:42:15,216 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f3526eb9bd778c7571f046690073138/B of 1f3526eb9bd778c7571f046690073138 into 4fa310472ee243a6b7521d1a0c8ece0e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:15,216 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:15,216 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/B, priority=12, startTime=1731645735189; duration=0sec 2024-11-15T04:42:15,216 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:15,216 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:B 2024-11-15T04:42:15,216 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:42:15,217 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:42:15,217 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/C is initiating minor compaction (all files) 2024-11-15T04:42:15,217 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/C in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:15,217 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/539ed8d4b91449baa621359003ed4a89, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/057a22b8c39f48c0b78b67066ca67b7e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/5986e170cba44e5084a455c2ced8ebb4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/87cee007f2074de88dea7c4cb32c009b] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=44.7 K 2024-11-15T04:42:15,217 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 539ed8d4b91449baa621359003ed4a89, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731645730063 2024-11-15T04:42:15,218 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 057a22b8c39f48c0b78b67066ca67b7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731645730727 2024-11-15T04:42:15,218 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 5986e170cba44e5084a455c2ced8ebb4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731645732102 2024-11-15T04:42:15,218 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 87cee007f2074de88dea7c4cb32c009b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645734322 2024-11-15T04:42:15,225 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#C#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:15,226 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/b7c869c27c3a4d27bcf7c1765fd96eff is 50, key is test_row_0/C:col10/1731645734328/Put/seqid=0 2024-11-15T04:42:15,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742384_1560 (size=12241) 2024-11-15T04:42:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:15,485 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:15,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:42:15,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b91c396a516841ca826764ef6921cc3c_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645734341/Put/seqid=0 2024-11-15T04:42:15,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742385_1561 (size=12204) 2024-11-15T04:42:15,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645795545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:15,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645795545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:15,613 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#A#compaction#471 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:15,613 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/db612b44f5c744e2bef00e4d54059359 is 175, key is test_row_0/A:col10/1731645734328/Put/seqid=0 2024-11-15T04:42:15,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742386_1562 (size=31195) 2024-11-15T04:42:15,633 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/b7c869c27c3a4d27bcf7c1765fd96eff as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/b7c869c27c3a4d27bcf7c1765fd96eff 2024-11-15T04:42:15,637 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f3526eb9bd778c7571f046690073138/C of 1f3526eb9bd778c7571f046690073138 into b7c869c27c3a4d27bcf7c1765fd96eff(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:15,637 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:15,637 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/C, priority=12, startTime=1731645735189; duration=0sec 2024-11-15T04:42:15,637 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:15,637 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:C 2024-11-15T04:42:15,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645795653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:15,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645795653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:15,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645795862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:15,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645795862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:15,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:15,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:15,897 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b91c396a516841ca826764ef6921cc3c_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b91c396a516841ca826764ef6921cc3c_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:15,899 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/fd0d822b7da149da913c5d55ec556257, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:15,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/fd0d822b7da149da913c5d55ec556257 is 175, key is test_row_0/A:col10/1731645734341/Put/seqid=0 2024-11-15T04:42:15,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742387_1563 (size=31005) 2024-11-15T04:42:15,903 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/fd0d822b7da149da913c5d55ec556257 2024-11-15T04:42:15,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/c8d9c4a619c344a2b638192a19a2943f is 50, key is test_row_0/B:col10/1731645734341/Put/seqid=0 2024-11-15T04:42:15,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742388_1564 (size=12051) 2024-11-15T04:42:15,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/c8d9c4a619c344a2b638192a19a2943f 2024-11-15T04:42:15,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/400bafc1f3f547b2a86545f6085dbd02 is 50, key is test_row_0/C:col10/1731645734341/Put/seqid=0 2024-11-15T04:42:15,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742389_1565 (size=12051) 2024-11-15T04:42:16,019 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/db612b44f5c744e2bef00e4d54059359 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/db612b44f5c744e2bef00e4d54059359 2024-11-15T04:42:16,023 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f3526eb9bd778c7571f046690073138/A of 1f3526eb9bd778c7571f046690073138 into db612b44f5c744e2bef00e4d54059359(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:16,023 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:16,023 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/A, priority=12, startTime=1731645735188; duration=0sec 2024-11-15T04:42:16,023 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:16,023 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:A 2024-11-15T04:42:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645796173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645796173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-15T04:42:16,192 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:16,193 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:42:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-15T04:42:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T04:42:16,194 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:42:16,195 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:42:16,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:42:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T04:42:16,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/400bafc1f3f547b2a86545f6085dbd02 2024-11-15T04:42:16,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/fd0d822b7da149da913c5d55ec556257 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/fd0d822b7da149da913c5d55ec556257 2024-11-15T04:42:16,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/fd0d822b7da149da913c5d55ec556257, entries=150, sequenceid=129, filesize=30.3 K 2024-11-15T04:42:16,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/c8d9c4a619c344a2b638192a19a2943f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c8d9c4a619c344a2b638192a19a2943f 2024-11-15T04:42:16,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c8d9c4a619c344a2b638192a19a2943f, entries=150, sequenceid=129, filesize=11.8 K 2024-11-15T04:42:16,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/400bafc1f3f547b2a86545f6085dbd02 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/400bafc1f3f547b2a86545f6085dbd02 2024-11-15T04:42:16,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/400bafc1f3f547b2a86545f6085dbd02, entries=150, sequenceid=129, filesize=11.8 K 2024-11-15T04:42:16,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1f3526eb9bd778c7571f046690073138 in 855ms, sequenceid=129, compaction requested=false 2024-11-15T04:42:16,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:16,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-15T04:42:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:16,347 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T04:42:16,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115df1959a597b14ad8911c588c5e6c0968_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645735539/Put/seqid=0 2024-11-15T04:42:16,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742390_1566 (size=12304) 2024-11-15T04:42:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T04:42:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:16,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:16,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645796696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:16,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645796697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:16,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:16,759 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115df1959a597b14ad8911c588c5e6c0968_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df1959a597b14ad8911c588c5e6c0968_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:16,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/9fdb1a380c034fc987d0117c0d097e1f, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:16,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/9fdb1a380c034fc987d0117c0d097e1f is 175, key is test_row_0/A:col10/1731645735539/Put/seqid=0 2024-11-15T04:42:16,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742391_1567 (size=31105) 2024-11-15T04:42:16,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:16,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645796803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:16,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645796803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:16,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T04:42:17,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645797012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:17,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645797013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:17,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:17,164 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/9fdb1a380c034fc987d0117c0d097e1f 2024-11-15T04:42:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/ec064d517cbe406b80383722af05ef5d is 50, key is test_row_0/B:col10/1731645735539/Put/seqid=0 2024-11-15T04:42:17,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742392_1568 (size=12151) 2024-11-15T04:42:17,173 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/ec064d517cbe406b80383722af05ef5d 2024-11-15T04:42:17,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/3d001bfe3b0f40849f10ee76644f9810 is 50, key is test_row_0/C:col10/1731645735539/Put/seqid=0 2024-11-15T04:42:17,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742393_1569 (size=12151) 2024-11-15T04:42:17,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645797323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:17,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645797323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T04:42:17,599 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/3d001bfe3b0f40849f10ee76644f9810 2024-11-15T04:42:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/9fdb1a380c034fc987d0117c0d097e1f as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/9fdb1a380c034fc987d0117c0d097e1f 2024-11-15T04:42:17,606 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/9fdb1a380c034fc987d0117c0d097e1f, entries=150, sequenceid=156, filesize=30.4 K 2024-11-15T04:42:17,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/ec064d517cbe406b80383722af05ef5d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/ec064d517cbe406b80383722af05ef5d 2024-11-15T04:42:17,610 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/ec064d517cbe406b80383722af05ef5d, entries=150, sequenceid=156, filesize=11.9 K 2024-11-15T04:42:17,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/3d001bfe3b0f40849f10ee76644f9810 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/3d001bfe3b0f40849f10ee76644f9810 2024-11-15T04:42:17,615 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/3d001bfe3b0f40849f10ee76644f9810, entries=150, sequenceid=156, filesize=11.9 K 2024-11-15T04:42:17,615 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1f3526eb9bd778c7571f046690073138 in 1268ms, sequenceid=156, compaction requested=true 2024-11-15T04:42:17,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:17,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:17,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-15T04:42:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-15T04:42:17,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-15T04:42:17,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2024-11-15T04:42:17,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.4250 sec 2024-11-15T04:42:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:17,834 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:17,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T04:42:17,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115930edf4f81e543c0a5d3398a050c1be1_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645736695/Put/seqid=0 2024-11-15T04:42:17,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742394_1570 (size=14794) 2024-11-15T04:42:17,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:17,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645797905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645797906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645798013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:18,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645798013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:18,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645798223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:18,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:18,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645798223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:18,243 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:18,246 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115930edf4f81e543c0a5d3398a050c1be1_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115930edf4f81e543c0a5d3398a050c1be1_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:18,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/06b0d017bea04463b5bf98ef200a2df4, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:18,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/06b0d017bea04463b5bf98ef200a2df4 is 175, key is test_row_0/A:col10/1731645736695/Put/seqid=0 2024-11-15T04:42:18,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742395_1571 (size=39749) 2024-11-15T04:42:18,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-15T04:42:18,343 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:18,343 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:42:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-15T04:42:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T04:42:18,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:42:18,345 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:42:18,345 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:42:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T04:42:18,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T04:42:18,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:18,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:18,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:18,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645798532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:18,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645798532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:18,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T04:42:18,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:18,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:18,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:18,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,651 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/06b0d017bea04463b5bf98ef200a2df4 2024-11-15T04:42:18,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/dc60881c356a4e998505b6c3eb42632a is 50, key is test_row_0/B:col10/1731645736695/Put/seqid=0 2024-11-15T04:42:18,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742396_1572 (size=12151) 2024-11-15T04:42:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T04:42:18,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T04:42:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:18,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645798912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:18,914 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:18,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645798922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:18,925 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:18,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645798943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:18,945 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:18,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:18,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T04:42:18,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:18,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:18,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:18,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T04:42:19,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645799043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:19,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645799043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:19,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/dc60881c356a4e998505b6c3eb42632a 2024-11-15T04:42:19,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/e8485818d8744dd098fc886adeddbeb0 is 50, key is test_row_0/C:col10/1731645736695/Put/seqid=0 2024-11-15T04:42:19,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742397_1573 (size=12151) 2024-11-15T04:42:19,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T04:42:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:19,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T04:42:19,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:19,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:19,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:19,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:19,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T04:42:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:19,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:19,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/e8485818d8744dd098fc886adeddbeb0 2024-11-15T04:42:19,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/06b0d017bea04463b5bf98ef200a2df4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/06b0d017bea04463b5bf98ef200a2df4 2024-11-15T04:42:19,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/06b0d017bea04463b5bf98ef200a2df4, entries=200, sequenceid=168, filesize=38.8 K 2024-11-15T04:42:19,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/dc60881c356a4e998505b6c3eb42632a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/dc60881c356a4e998505b6c3eb42632a 2024-11-15T04:42:19,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/dc60881c356a4e998505b6c3eb42632a, entries=150, sequenceid=168, filesize=11.9 K 2024-11-15T04:42:19,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/e8485818d8744dd098fc886adeddbeb0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e8485818d8744dd098fc886adeddbeb0 2024-11-15T04:42:19,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e8485818d8744dd098fc886adeddbeb0, entries=150, sequenceid=168, filesize=11.9 K 2024-11-15T04:42:19,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1f3526eb9bd778c7571f046690073138 in 1647ms, sequenceid=168, compaction requested=true 2024-11-15T04:42:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:42:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:42:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:19,481 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:42:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:42:19,481 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:42:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:19,482 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:42:19,482 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/B is initiating minor compaction (all files) 2024-11-15T04:42:19,482 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:42:19,482 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/B in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,482 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/A is initiating minor compaction (all files) 2024-11-15T04:42:19,482 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/A in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,482 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/4fa310472ee243a6b7521d1a0c8ece0e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c8d9c4a619c344a2b638192a19a2943f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/ec064d517cbe406b80383722af05ef5d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/dc60881c356a4e998505b6c3eb42632a] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=47.5 K 2024-11-15T04:42:19,482 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/db612b44f5c744e2bef00e4d54059359, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/fd0d822b7da149da913c5d55ec556257, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/9fdb1a380c034fc987d0117c0d097e1f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/06b0d017bea04463b5bf98ef200a2df4] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=129.9 K 2024-11-15T04:42:19,482 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,482 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/db612b44f5c744e2bef00e4d54059359, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/fd0d822b7da149da913c5d55ec556257, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/9fdb1a380c034fc987d0117c0d097e1f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/06b0d017bea04463b5bf98ef200a2df4] 2024-11-15T04:42:19,482 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fa310472ee243a6b7521d1a0c8ece0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645734322 2024-11-15T04:42:19,482 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting db612b44f5c744e2bef00e4d54059359, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645734322 2024-11-15T04:42:19,482 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c8d9c4a619c344a2b638192a19a2943f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731645734341 2024-11-15T04:42:19,482 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd0d822b7da149da913c5d55ec556257, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731645734341 2024-11-15T04:42:19,483 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting ec064d517cbe406b80383722af05ef5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731645735539 2024-11-15T04:42:19,483 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fdb1a380c034fc987d0117c0d097e1f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731645735539 2024-11-15T04:42:19,483 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting dc60881c356a4e998505b6c3eb42632a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731645736692 2024-11-15T04:42:19,483 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06b0d017bea04463b5bf98ef200a2df4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731645736692 2024-11-15T04:42:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T04:42:19,487 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:19,488 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115072d1a322248429486589935ea1c6d22_1f3526eb9bd778c7571f046690073138 store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:19,489 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#B#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:19,489 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/b8802c1088464e63a28265205a54d291 is 50, key is test_row_0/B:col10/1731645736695/Put/seqid=0 2024-11-15T04:42:19,490 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115072d1a322248429486589935ea1c6d22_1f3526eb9bd778c7571f046690073138, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:19,490 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115072d1a322248429486589935ea1c6d22_1f3526eb9bd778c7571f046690073138 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:19,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742398_1574 (size=12527) 2024-11-15T04:42:19,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742399_1575 (size=4469) 2024-11-15T04:42:19,497 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/b8802c1088464e63a28265205a54d291 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/b8802c1088464e63a28265205a54d291 2024-11-15T04:42:19,500 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f3526eb9bd778c7571f046690073138/B of 1f3526eb9bd778c7571f046690073138 into b8802c1088464e63a28265205a54d291(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:19,501 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:19,501 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/B, priority=12, startTime=1731645739481; duration=0sec 2024-11-15T04:42:19,501 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:19,501 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:B 2024-11-15T04:42:19,501 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:42:19,502 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:42:19,502 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/C is initiating minor compaction (all files) 2024-11-15T04:42:19,502 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/C in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,502 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/b7c869c27c3a4d27bcf7c1765fd96eff, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/400bafc1f3f547b2a86545f6085dbd02, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/3d001bfe3b0f40849f10ee76644f9810, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e8485818d8744dd098fc886adeddbeb0] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=47.5 K 2024-11-15T04:42:19,502 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting b7c869c27c3a4d27bcf7c1765fd96eff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731645734322 2024-11-15T04:42:19,503 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 400bafc1f3f547b2a86545f6085dbd02, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731645734341 2024-11-15T04:42:19,503 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d001bfe3b0f40849f10ee76644f9810, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731645735539 2024-11-15T04:42:19,503 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e8485818d8744dd098fc886adeddbeb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731645736692 2024-11-15T04:42:19,510 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#C#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:19,511 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/72dff95eaea84333b64d213066f498cb is 50, key is test_row_0/C:col10/1731645736695/Put/seqid=0 2024-11-15T04:42:19,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742400_1576 (size=12527) 2024-11-15T04:42:19,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T04:42:19,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:19,565 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T04:42:19,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158e1911f988b04eb2a9f2ea3defaf8fea_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645737904/Put/seqid=0 2024-11-15T04:42:19,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742401_1577 (size=12304) 2024-11-15T04:42:19,894 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#A#compaction#483 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:19,894 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/ea5e3bc03bec4211b12a99c0838a4d2d is 175, key is test_row_0/A:col10/1731645736695/Put/seqid=0 2024-11-15T04:42:19,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742402_1578 (size=31481) 2024-11-15T04:42:19,918 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/72dff95eaea84333b64d213066f498cb as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/72dff95eaea84333b64d213066f498cb 2024-11-15T04:42:19,922 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f3526eb9bd778c7571f046690073138/C of 1f3526eb9bd778c7571f046690073138 into 72dff95eaea84333b64d213066f498cb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:19,922 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:19,922 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/C, priority=12, startTime=1731645739481; duration=0sec 2024-11-15T04:42:19,922 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:19,922 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:C 2024-11-15T04:42:19,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:19,981 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158e1911f988b04eb2a9f2ea3defaf8fea_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158e1911f988b04eb2a9f2ea3defaf8fea_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/560276a83bce4053bc5d0136212ce40c, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/560276a83bce4053bc5d0136212ce40c is 175, key is test_row_0/A:col10/1731645737904/Put/seqid=0 2024-11-15T04:42:19,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742403_1579 (size=31105) 2024-11-15T04:42:20,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:20,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645800072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:20,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645800075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:20,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:20,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645800182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:20,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645800182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:20,301 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/ea5e3bc03bec4211b12a99c0838a4d2d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ea5e3bc03bec4211b12a99c0838a4d2d 2024-11-15T04:42:20,304 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f3526eb9bd778c7571f046690073138/A of 1f3526eb9bd778c7571f046690073138 into ea5e3bc03bec4211b12a99c0838a4d2d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:20,304 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:20,304 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/A, priority=12, startTime=1731645739481; duration=0sec 2024-11-15T04:42:20,304 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:20,304 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:A 2024-11-15T04:42:20,386 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/560276a83bce4053bc5d0136212ce40c 2024-11-15T04:42:20,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/c6ff4589eecf403c966eda6fe8d29f3c is 50, key is test_row_0/B:col10/1731645737904/Put/seqid=0 2024-11-15T04:42:20,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742404_1580 (size=12151) 2024-11-15T04:42:20,395 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/c6ff4589eecf403c966eda6fe8d29f3c 2024-11-15T04:42:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645800393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645800393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:20,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/0425514a4a3843518a9f1354a48651c4 is 50, key is test_row_0/C:col10/1731645737904/Put/seqid=0 2024-11-15T04:42:20,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742405_1581 (size=12151) 2024-11-15T04:42:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T04:42:20,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645800703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:20,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:20,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645800703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:20,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:20,803 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/0425514a4a3843518a9f1354a48651c4 2024-11-15T04:42:20,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/560276a83bce4053bc5d0136212ce40c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/560276a83bce4053bc5d0136212ce40c 2024-11-15T04:42:20,809 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/560276a83bce4053bc5d0136212ce40c, entries=150, sequenceid=193, filesize=30.4 K 2024-11-15T04:42:20,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/c6ff4589eecf403c966eda6fe8d29f3c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c6ff4589eecf403c966eda6fe8d29f3c 2024-11-15T04:42:20,812 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c6ff4589eecf403c966eda6fe8d29f3c, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T04:42:20,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/0425514a4a3843518a9f1354a48651c4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/0425514a4a3843518a9f1354a48651c4 2024-11-15T04:42:20,817 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/0425514a4a3843518a9f1354a48651c4, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T04:42:20,818 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1f3526eb9bd778c7571f046690073138 in 1254ms, sequenceid=193, compaction requested=false 2024-11-15T04:42:20,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:20,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:20,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-15T04:42:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-15T04:42:20,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-15T04:42:20,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4740 sec 2024-11-15T04:42:20,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.4780 sec 2024-11-15T04:42:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:21,215 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:21,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T04:42:21,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152c134572d4974f42b288cc5753d036d1_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645740070/Put/seqid=0 2024-11-15T04:42:21,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742406_1582 (size=14794) 2024-11-15T04:42:21,224 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:21,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152c134572d4974f42b288cc5753d036d1_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c134572d4974f42b288cc5753d036d1_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:21,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/4f4467e715cd4a9fbba4cde68b5ccb2c, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:21,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/4f4467e715cd4a9fbba4cde68b5ccb2c is 175, key is test_row_0/A:col10/1731645740070/Put/seqid=0 2024-11-15T04:42:21,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742407_1583 (size=39749) 2024-11-15T04:42:21,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645801269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:21,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645801269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:21,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645801383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:21,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645801383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:21,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645801592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:21,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645801592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:21,632 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/4f4467e715cd4a9fbba4cde68b5ccb2c 2024-11-15T04:42:21,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/35a7e27c9dcf41578a357e81e4a673ea is 50, key is test_row_0/B:col10/1731645740070/Put/seqid=0 2024-11-15T04:42:21,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742408_1584 (size=12151) 2024-11-15T04:42:21,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/35a7e27c9dcf41578a357e81e4a673ea 2024-11-15T04:42:21,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/e27c36c1e414458e8d12aab2b4f91566 is 50, key is test_row_0/C:col10/1731645740070/Put/seqid=0 2024-11-15T04:42:21,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742409_1585 (size=12151) 2024-11-15T04:42:21,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645801903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645801903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:22,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/e27c36c1e414458e8d12aab2b4f91566 2024-11-15T04:42:22,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/4f4467e715cd4a9fbba4cde68b5ccb2c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/4f4467e715cd4a9fbba4cde68b5ccb2c 2024-11-15T04:42:22,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/4f4467e715cd4a9fbba4cde68b5ccb2c, entries=200, sequenceid=209, filesize=38.8 K 2024-11-15T04:42:22,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/35a7e27c9dcf41578a357e81e4a673ea as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/35a7e27c9dcf41578a357e81e4a673ea 2024-11-15T04:42:22,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/35a7e27c9dcf41578a357e81e4a673ea, entries=150, sequenceid=209, filesize=11.9 K 2024-11-15T04:42:22,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/e27c36c1e414458e8d12aab2b4f91566 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e27c36c1e414458e8d12aab2b4f91566 2024-11-15T04:42:22,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e27c36c1e414458e8d12aab2b4f91566, entries=150, sequenceid=209, filesize=11.9 K 2024-11-15T04:42:22,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 1f3526eb9bd778c7571f046690073138 in 847ms, sequenceid=209, compaction requested=true 2024-11-15T04:42:22,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:22,062 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:42:22,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:22,063 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:42:22,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:42:22,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:22,063 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:42:22,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:42:22,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:22,063 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:42:22,063 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:42:22,063 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/B is initiating minor compaction (all files) 2024-11-15T04:42:22,063 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/A is initiating minor compaction (all files) 2024-11-15T04:42:22,064 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/B in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:22,064 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/A in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:22,064 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/b8802c1088464e63a28265205a54d291, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c6ff4589eecf403c966eda6fe8d29f3c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/35a7e27c9dcf41578a357e81e4a673ea] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=36.0 K 2024-11-15T04:42:22,064 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ea5e3bc03bec4211b12a99c0838a4d2d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/560276a83bce4053bc5d0136212ce40c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/4f4467e715cd4a9fbba4cde68b5ccb2c] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=99.9 K 2024-11-15T04:42:22,064 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:22,064 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ea5e3bc03bec4211b12a99c0838a4d2d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/560276a83bce4053bc5d0136212ce40c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/4f4467e715cd4a9fbba4cde68b5ccb2c] 2024-11-15T04:42:22,064 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting b8802c1088464e63a28265205a54d291, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731645736692 2024-11-15T04:42:22,064 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea5e3bc03bec4211b12a99c0838a4d2d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731645736692 2024-11-15T04:42:22,064 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting c6ff4589eecf403c966eda6fe8d29f3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731645737862 2024-11-15T04:42:22,064 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 560276a83bce4053bc5d0136212ce40c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731645737862 2024-11-15T04:42:22,064 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 35a7e27c9dcf41578a357e81e4a673ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731645740070 2024-11-15T04:42:22,064 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f4467e715cd4a9fbba4cde68b5ccb2c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731645740067 2024-11-15T04:42:22,068 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:22,070 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#B#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:22,070 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111559078f94360d441f8c19ba9b83358eee_1f3526eb9bd778c7571f046690073138 store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:22,070 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/9d9e0011589b42a4965c2a6bef8d02a0 is 50, key is test_row_0/B:col10/1731645740070/Put/seqid=0 2024-11-15T04:42:22,072 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111559078f94360d441f8c19ba9b83358eee_1f3526eb9bd778c7571f046690073138, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:22,072 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111559078f94360d441f8c19ba9b83358eee_1f3526eb9bd778c7571f046690073138 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:22,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742410_1586 (size=12629) 2024-11-15T04:42:22,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742411_1587 (size=4469) 2024-11-15T04:42:22,078 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/9d9e0011589b42a4965c2a6bef8d02a0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/9d9e0011589b42a4965c2a6bef8d02a0 2024-11-15T04:42:22,081 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f3526eb9bd778c7571f046690073138/B of 1f3526eb9bd778c7571f046690073138 into 9d9e0011589b42a4965c2a6bef8d02a0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:22,081 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:22,081 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/B, priority=13, startTime=1731645742063; duration=0sec 2024-11-15T04:42:22,081 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:22,082 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:B 2024-11-15T04:42:22,082 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:42:22,082 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:42:22,082 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/C is initiating minor compaction (all files) 2024-11-15T04:42:22,082 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/C in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:22,082 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/72dff95eaea84333b64d213066f498cb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/0425514a4a3843518a9f1354a48651c4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e27c36c1e414458e8d12aab2b4f91566] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=36.0 K 2024-11-15T04:42:22,083 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 72dff95eaea84333b64d213066f498cb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731645736692 2024-11-15T04:42:22,083 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 0425514a4a3843518a9f1354a48651c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731645737862 2024-11-15T04:42:22,083 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e27c36c1e414458e8d12aab2b4f91566, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731645740070 2024-11-15T04:42:22,088 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#C#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:22,088 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/b7bf73ed6e77444f84cc81ff7163aa6b is 50, key is test_row_0/C:col10/1731645740070/Put/seqid=0 2024-11-15T04:42:22,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742412_1588 (size=12629) 2024-11-15T04:42:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:22,415 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:22,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T04:42:22,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156ac020104dfe4af7bc7d0b1f9dd8fcd1_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645742415/Put/seqid=0 2024-11-15T04:42:22,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742413_1589 (size=14794) 2024-11-15T04:42:22,426 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:22,428 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156ac020104dfe4af7bc7d0b1f9dd8fcd1_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156ac020104dfe4af7bc7d0b1f9dd8fcd1_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:22,429 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/43774f22cc5844fcaf52212b120a3bb4, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:22,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/43774f22cc5844fcaf52212b120a3bb4 is 175, key is test_row_0/A:col10/1731645742415/Put/seqid=0 2024-11-15T04:42:22,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742414_1590 (size=39749) 2024-11-15T04:42:22,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645802433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645802434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:22,477 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:22,478 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/8a4901e92c114594b8cf56f5fc894c9b is 175, key is test_row_0/A:col10/1731645740070/Put/seqid=0 2024-11-15T04:42:22,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742415_1591 (size=31583) 2024-11-15T04:42:22,495 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/b7bf73ed6e77444f84cc81ff7163aa6b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/b7bf73ed6e77444f84cc81ff7163aa6b 2024-11-15T04:42:22,499 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f3526eb9bd778c7571f046690073138/C of 1f3526eb9bd778c7571f046690073138 into b7bf73ed6e77444f84cc81ff7163aa6b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:22,499 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:22,499 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/C, priority=13, startTime=1731645742063; duration=0sec 2024-11-15T04:42:22,499 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:22,499 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:C 2024-11-15T04:42:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T04:42:22,513 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:22,513 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:42:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-15T04:42:22,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:42:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T04:42:22,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:42:22,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:42:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645802542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:22,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645802542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,544 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T04:42:22,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T04:42:22,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:22,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:22,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:22,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:22,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:22,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645802753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:22,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645802753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:22,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T04:42:22,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:22,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:22,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:22,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:22,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:22,833 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/43774f22cc5844fcaf52212b120a3bb4 2024-11-15T04:42:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T04:42:22,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/42df6a7308cf42a0916f83de72b1b72d is 50, key is test_row_0/B:col10/1731645742415/Put/seqid=0 2024-11-15T04:42:22,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742416_1592 (size=12151) 2024-11-15T04:42:22,885 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/8a4901e92c114594b8cf56f5fc894c9b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8a4901e92c114594b8cf56f5fc894c9b 2024-11-15T04:42:22,889 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f3526eb9bd778c7571f046690073138/A of 1f3526eb9bd778c7571f046690073138 into 8a4901e92c114594b8cf56f5fc894c9b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:22,889 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:22,889 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/A, priority=13, startTime=1731645742062; duration=0sec 2024-11-15T04:42:22,889 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:22,889 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:A 2024-11-15T04:42:22,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T04:42:22,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:22,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:22,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:22,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:22,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:23,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645803062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:23,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:23,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645803063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:23,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:23,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T04:42:23,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:23,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:23,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:23,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:23,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:23,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:23,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T04:42:23,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/42df6a7308cf42a0916f83de72b1b72d 2024-11-15T04:42:23,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/29c5eab2342a4504856653e6b97f96ff is 50, key is test_row_0/C:col10/1731645742415/Put/seqid=0 2024-11-15T04:42:23,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742417_1593 (size=12151) 2024-11-15T04:42:23,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/29c5eab2342a4504856653e6b97f96ff 2024-11-15T04:42:23,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/43774f22cc5844fcaf52212b120a3bb4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/43774f22cc5844fcaf52212b120a3bb4 2024-11-15T04:42:23,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/43774f22cc5844fcaf52212b120a3bb4, entries=200, sequenceid=235, filesize=38.8 K 2024-11-15T04:42:23,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/42df6a7308cf42a0916f83de72b1b72d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/42df6a7308cf42a0916f83de72b1b72d 2024-11-15T04:42:23,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/42df6a7308cf42a0916f83de72b1b72d, entries=150, sequenceid=235, filesize=11.9 K 2024-11-15T04:42:23,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/29c5eab2342a4504856653e6b97f96ff as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/29c5eab2342a4504856653e6b97f96ff 2024-11-15T04:42:23,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/29c5eab2342a4504856653e6b97f96ff, entries=150, sequenceid=235, filesize=11.9 K 2024-11-15T04:42:23,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1f3526eb9bd778c7571f046690073138 in 861ms, sequenceid=235, compaction requested=false 2024-11-15T04:42:23,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:23,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T04:42:23,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:23,277 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:42:23,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151b3c343f6e7c41608d21061a5b4f4023_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645742428/Put/seqid=0 2024-11-15T04:42:23,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742418_1594 (size=12304) 2024-11-15T04:42:23,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:23,295 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151b3c343f6e7c41608d21061a5b4f4023_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151b3c343f6e7c41608d21061a5b4f4023_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:23,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/486d595306c14f049612a88e09da7a65, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:23,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/486d595306c14f049612a88e09da7a65 is 175, key is test_row_0/A:col10/1731645742428/Put/seqid=0 2024-11-15T04:42:23,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742419_1595 (size=31105) 2024-11-15T04:42:23,312 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/486d595306c14f049612a88e09da7a65 2024-11-15T04:42:23,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/d1253d2dbbba4b1a84a3640ae33f321a is 50, key is test_row_0/B:col10/1731645742428/Put/seqid=0 2024-11-15T04:42:23,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742420_1596 (size=12151) 2024-11-15T04:42:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:23,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:23,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645803645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:23,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:23,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645803645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T04:42:23,741 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/d1253d2dbbba4b1a84a3640ae33f321a 2024-11-15T04:42:23,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/97053b1700b14b559ded82c3b8ec15a7 is 50, key is test_row_0/C:col10/1731645742428/Put/seqid=0 2024-11-15T04:42:23,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742421_1597 (size=12151) 2024-11-15T04:42:23,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645803753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:23,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645803753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645803963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645803963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:23,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:24,149 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/97053b1700b14b559ded82c3b8ec15a7 2024-11-15T04:42:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/486d595306c14f049612a88e09da7a65 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/486d595306c14f049612a88e09da7a65 2024-11-15T04:42:24,154 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/486d595306c14f049612a88e09da7a65, entries=150, sequenceid=248, filesize=30.4 K 2024-11-15T04:42:24,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/d1253d2dbbba4b1a84a3640ae33f321a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/d1253d2dbbba4b1a84a3640ae33f321a 2024-11-15T04:42:24,158 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/d1253d2dbbba4b1a84a3640ae33f321a, entries=150, sequenceid=248, filesize=11.9 K 2024-11-15T04:42:24,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/97053b1700b14b559ded82c3b8ec15a7 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/97053b1700b14b559ded82c3b8ec15a7 2024-11-15T04:42:24,161 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/97053b1700b14b559ded82c3b8ec15a7, entries=150, sequenceid=248, filesize=11.9 K 2024-11-15T04:42:24,161 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1f3526eb9bd778c7571f046690073138 in 884ms, sequenceid=248, compaction requested=true 2024-11-15T04:42:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:24,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-15T04:42:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-15T04:42:24,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-15T04:42:24,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6480 sec 2024-11-15T04:42:24,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.6510 sec 2024-11-15T04:42:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:24,276 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:24,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T04:42:24,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154d4efbc2a97e4ccdabb02ce5eab4aeef_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645743638/Put/seqid=0 2024-11-15T04:42:24,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742422_1598 (size=14994) 2024-11-15T04:42:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645804290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645804294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:24,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:24,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645804402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645804402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:24,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645804613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:24,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645804613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T04:42:24,663 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:24,663 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-15T04:42:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-15T04:42:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T04:42:24,665 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T04:42:24,665 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T04:42:24,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T04:42:24,686 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:24,689 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154d4efbc2a97e4ccdabb02ce5eab4aeef_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154d4efbc2a97e4ccdabb02ce5eab4aeef_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:24,689 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/ed593f06ee7044dd931da7106501645b, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:24,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/ed593f06ee7044dd931da7106501645b is 175, key is test_row_0/A:col10/1731645743638/Put/seqid=0 2024-11-15T04:42:24,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742423_1599 (size=39949) 2024-11-15T04:42:24,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T04:42:24,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T04:42:24,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:24,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:24,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:24,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:24,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:24,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:24,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645804923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645804923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:24,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T04:42:24,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:24,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:24,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:24,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:24,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T04:42:25,093 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/ed593f06ee7044dd931da7106501645b 2024-11-15T04:42:25,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/1e41b8782bed40a786394d5c25813d32 is 50, key is test_row_0/B:col10/1731645743638/Put/seqid=0 2024-11-15T04:42:25,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742424_1600 (size=12301) 2024-11-15T04:42:25,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T04:42:25,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:25,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T04:42:25,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:25,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T04:42:25,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T04:42:25,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:25,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645805432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:25,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:25,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:25,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645805433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:25,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:25,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:25,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:25,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:25,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/1e41b8782bed40a786394d5c25813d32 2024-11-15T04:42:25,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/8f8dd196e2f64a238f2f3f8bf0ba3de1 is 50, key is test_row_0/C:col10/1731645743638/Put/seqid=0 2024-11-15T04:42:25,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742425_1601 (size=12301) 2024-11-15T04:42:25,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T04:42:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T04:42:25,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:25,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T04:42:25,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T04:42:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T04:42:25,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/8f8dd196e2f64a238f2f3f8bf0ba3de1 2024-11-15T04:42:25,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/ed593f06ee7044dd931da7106501645b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ed593f06ee7044dd931da7106501645b 2024-11-15T04:42:25,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ed593f06ee7044dd931da7106501645b, entries=200, sequenceid=275, filesize=39.0 K 2024-11-15T04:42:25,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/1e41b8782bed40a786394d5c25813d32 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/1e41b8782bed40a786394d5c25813d32 2024-11-15T04:42:25,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/1e41b8782bed40a786394d5c25813d32, entries=150, sequenceid=275, filesize=12.0 K 2024-11-15T04:42:25,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/8f8dd196e2f64a238f2f3f8bf0ba3de1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/8f8dd196e2f64a238f2f3f8bf0ba3de1 2024-11-15T04:42:25,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/8f8dd196e2f64a238f2f3f8bf0ba3de1, entries=150, sequenceid=275, filesize=12.0 K 2024-11-15T04:42:25,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1f3526eb9bd778c7571f046690073138 in 1650ms, sequenceid=275, compaction requested=true 2024-11-15T04:42:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:42:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:25,926 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:42:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:42:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:25,926 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:42:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:42:25,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:25,927 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:42:25,927 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/B is initiating minor compaction (all files) 2024-11-15T04:42:25,927 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/B in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,927 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/9d9e0011589b42a4965c2a6bef8d02a0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/42df6a7308cf42a0916f83de72b1b72d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/d1253d2dbbba4b1a84a3640ae33f321a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/1e41b8782bed40a786394d5c25813d32] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=48.1 K 2024-11-15T04:42:25,927 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:42:25,927 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/A is initiating minor compaction (all files) 2024-11-15T04:42:25,927 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/A in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,927 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8a4901e92c114594b8cf56f5fc894c9b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/43774f22cc5844fcaf52212b120a3bb4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/486d595306c14f049612a88e09da7a65, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ed593f06ee7044dd931da7106501645b] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=139.0 K 2024-11-15T04:42:25,927 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,927 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8a4901e92c114594b8cf56f5fc894c9b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/43774f22cc5844fcaf52212b120a3bb4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/486d595306c14f049612a88e09da7a65, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ed593f06ee7044dd931da7106501645b] 2024-11-15T04:42:25,928 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d9e0011589b42a4965c2a6bef8d02a0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731645740070 2024-11-15T04:42:25,928 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a4901e92c114594b8cf56f5fc894c9b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731645740070 2024-11-15T04:42:25,928 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 42df6a7308cf42a0916f83de72b1b72d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731645741267 2024-11-15T04:42:25,928 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43774f22cc5844fcaf52212b120a3bb4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731645741267 2024-11-15T04:42:25,928 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting d1253d2dbbba4b1a84a3640ae33f321a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731645742428 2024-11-15T04:42:25,928 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 486d595306c14f049612a88e09da7a65, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731645742428 2024-11-15T04:42:25,928 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e41b8782bed40a786394d5c25813d32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731645743638 2024-11-15T04:42:25,928 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed593f06ee7044dd931da7106501645b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731645743638 2024-11-15T04:42:25,934 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:25,935 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/575401ee82e5430bac6bf5d3261e51a1 is 50, key is test_row_0/B:col10/1731645743638/Put/seqid=0 2024-11-15T04:42:25,935 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:25,942 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411154f0b2f46931b402aa686291fdbf86d86_1f3526eb9bd778c7571f046690073138 store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:25,944 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411154f0b2f46931b402aa686291fdbf86d86_1f3526eb9bd778c7571f046690073138, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:25,945 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154f0b2f46931b402aa686291fdbf86d86_1f3526eb9bd778c7571f046690073138 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:25,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742426_1602 (size=12915) 2024-11-15T04:42:25,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742427_1603 (size=4469) 2024-11-15T04:42:25,957 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#A#compaction#504 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:25,957 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/eeea2dfe69e34852b81e1de7b82a35a5 is 175, key is test_row_0/A:col10/1731645743638/Put/seqid=0 2024-11-15T04:42:25,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742428_1604 (size=31869) 2024-11-15T04:42:25,964 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/eeea2dfe69e34852b81e1de7b82a35a5 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/eeea2dfe69e34852b81e1de7b82a35a5 2024-11-15T04:42:25,968 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f3526eb9bd778c7571f046690073138/A of 1f3526eb9bd778c7571f046690073138 into eeea2dfe69e34852b81e1de7b82a35a5(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:25,968 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:25,968 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/A, priority=12, startTime=1731645745926; duration=0sec 2024-11-15T04:42:25,969 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:25,969 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:A 2024-11-15T04:42:25,969 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T04:42:25,969 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T04:42:25,969 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/C is initiating minor compaction (all files) 2024-11-15T04:42:25,969 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/C in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:25,969 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/b7bf73ed6e77444f84cc81ff7163aa6b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/29c5eab2342a4504856653e6b97f96ff, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/97053b1700b14b559ded82c3b8ec15a7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/8f8dd196e2f64a238f2f3f8bf0ba3de1] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=48.1 K 2024-11-15T04:42:25,970 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7bf73ed6e77444f84cc81ff7163aa6b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731645740070 2024-11-15T04:42:25,970 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29c5eab2342a4504856653e6b97f96ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731645741267 2024-11-15T04:42:25,970 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97053b1700b14b559ded82c3b8ec15a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731645742428 2024-11-15T04:42:25,970 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f8dd196e2f64a238f2f3f8bf0ba3de1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731645743638 2024-11-15T04:42:25,976 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#C#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:25,976 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/f81b0a622e124fab94daa6854307a354 is 50, key is test_row_0/C:col10/1731645743638/Put/seqid=0 2024-11-15T04:42:25,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742429_1605 (size=12915) 2024-11-15T04:42:26,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42935 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T04:42:26,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:26,038 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T04:42:26,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ee8c05a0ec754a50a19c447b654cd261_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645744293/Put/seqid=0 2024-11-15T04:42:26,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742430_1606 (size=12454) 2024-11-15T04:42:26,350 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/575401ee82e5430bac6bf5d3261e51a1 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/575401ee82e5430bac6bf5d3261e51a1 2024-11-15T04:42:26,353 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f3526eb9bd778c7571f046690073138/B of 1f3526eb9bd778c7571f046690073138 into 575401ee82e5430bac6bf5d3261e51a1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:26,353 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:26,353 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/B, priority=12, startTime=1731645745926; duration=0sec 2024-11-15T04:42:26,353 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:26,353 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:B 2024-11-15T04:42:26,395 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/f81b0a622e124fab94daa6854307a354 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/f81b0a622e124fab94daa6854307a354 2024-11-15T04:42:26,402 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1f3526eb9bd778c7571f046690073138/C of 1f3526eb9bd778c7571f046690073138 into f81b0a622e124fab94daa6854307a354(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:26,402 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:26,402 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/C, priority=12, startTime=1731645745926; duration=0sec 2024-11-15T04:42:26,402 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:26,402 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:C 2024-11-15T04:42:26,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:26,451 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ee8c05a0ec754a50a19c447b654cd261_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ee8c05a0ec754a50a19c447b654cd261_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:26,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/42ab5ed4c17b4bb48cee7798f0a481ff, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:26,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/42ab5ed4c17b4bb48cee7798f0a481ff is 175, key is test_row_0/A:col10/1731645744293/Put/seqid=0 2024-11-15T04:42:26,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742431_1607 (size=31255) 2024-11-15T04:42:26,455 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/42ab5ed4c17b4bb48cee7798f0a481ff 2024-11-15T04:42:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:26,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. as already flushing 2024-11-15T04:42:26,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/25d6391763d74a8f8e4c51b038ec57a3 is 50, key is test_row_0/B:col10/1731645744293/Put/seqid=0 2024-11-15T04:42:26,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742432_1608 (size=12301) 2024-11-15T04:42:26,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645806497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:26,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645806500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:26,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645806602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:26,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645806612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T04:42:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645806813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:26,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645806822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:26,865 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/25d6391763d74a8f8e4c51b038ec57a3 2024-11-15T04:42:26,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/d4dad1e206294bd595b1ffcc3e343c6d is 50, key is test_row_0/C:col10/1731645744293/Put/seqid=0 2024-11-15T04:42:26,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742433_1609 (size=12301) 2024-11-15T04:42:27,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:27,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645807122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:27,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645807133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:27,273 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/d4dad1e206294bd595b1ffcc3e343c6d 2024-11-15T04:42:27,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/42ab5ed4c17b4bb48cee7798f0a481ff as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/42ab5ed4c17b4bb48cee7798f0a481ff 2024-11-15T04:42:27,278 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/42ab5ed4c17b4bb48cee7798f0a481ff, entries=150, sequenceid=285, filesize=30.5 K 2024-11-15T04:42:27,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/25d6391763d74a8f8e4c51b038ec57a3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/25d6391763d74a8f8e4c51b038ec57a3 2024-11-15T04:42:27,282 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/25d6391763d74a8f8e4c51b038ec57a3, entries=150, sequenceid=285, filesize=12.0 K 2024-11-15T04:42:27,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/d4dad1e206294bd595b1ffcc3e343c6d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/d4dad1e206294bd595b1ffcc3e343c6d 2024-11-15T04:42:27,285 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/d4dad1e206294bd595b1ffcc3e343c6d, entries=150, sequenceid=285, filesize=12.0 K 2024-11-15T04:42:27,285 INFO [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 1f3526eb9bd778c7571f046690073138 in 1247ms, sequenceid=285, compaction requested=false 2024-11-15T04:42:27,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:27,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:27,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/26747e6ada4f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-15T04:42:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-15T04:42:27,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-15T04:42:27,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6210 sec 2024-11-15T04:42:27,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.6250 sec 2024-11-15T04:42:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:27,635 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:27,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T04:42:27,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115247bbc33aace4486930a6a5623990d2b_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645747634/Put/seqid=0 2024-11-15T04:42:27,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742434_1610 (size=12454) 2024-11-15T04:42:27,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:27,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645807648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:27,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645807649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:27,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645807752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:27,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:27,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645807762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:27,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:27,854 INFO [Thread-2377 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2377. 2024-11-15T04:42:27,854 DEBUG [Thread-2377 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:27,854 DEBUG [Thread-2377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:27,854 DEBUG [Thread-2377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:27,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:27,854 INFO [Thread-2369 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2369. 2024-11-15T04:42:27,855 DEBUG [Thread-2369 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:27,855 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:27,855 DEBUG [Thread-2369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:27,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:27,855 INFO [Thread-2371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2371. 2024-11-15T04:42:27,855 DEBUG [Thread-2371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:27,855 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:27,856 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:27,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:27,857 INFO [Thread-2373 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2373. 2024-11-15T04:42:27,857 DEBUG [Thread-2373 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:27,857 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:27,857 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:27,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:27,857 INFO [Thread-2375 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2375. 2024-11-15T04:42:27,857 DEBUG [Thread-2375 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:27,857 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:27,857 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:27,857 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:27,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645807963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:27,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645807973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:28,049 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:28,058 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115247bbc33aace4486930a6a5623990d2b_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115247bbc33aace4486930a6a5623990d2b_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:28,059 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/20783c5a57b741f0a37827c7c8ee6ceb, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:28,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/20783c5a57b741f0a37827c7c8ee6ceb is 175, key is test_row_0/A:col10/1731645747634/Put/seqid=0 2024-11-15T04:42:28,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742435_1611 (size=31255) 2024-11-15T04:42:28,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645808273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:28,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645808283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:28,465 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/20783c5a57b741f0a37827c7c8ee6ceb 2024-11-15T04:42:28,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/e162ed9938024ef1af68b6626fc79739 is 50, key is test_row_0/B:col10/1731645747634/Put/seqid=0 2024-11-15T04:42:28,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742436_1612 (size=12301) 2024-11-15T04:42:28,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38834 deadline: 1731645808783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:28,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:28,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38794 deadline: 1731645808793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T04:42:28,834 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:28,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/e162ed9938024ef1af68b6626fc79739 2024-11-15T04:42:28,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/dd56825b3e0f4ba695a745061f5673b3 is 50, key is test_row_0/C:col10/1731645747634/Put/seqid=0 2024-11-15T04:42:28,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742437_1613 (size=12301) 2024-11-15T04:42:29,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:29,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38804 deadline: 1731645809003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:29,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38818 deadline: 1731645809003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:29,005 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_2' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:29,005 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_1' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:29,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T04:42:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38828 deadline: 1731645809013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:29,014 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 26747e6ada4f,42935,1731645581086 for 'test_row_0' in 1f3526eb9bd778c7571f046690073138 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18290 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T04:42:29,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 , the old value is region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:29,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T04:42:29,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., hostname=26747e6ada4f,42935,1731645581086, seqNum=5 because the exception is null or not the one we care about 2024-11-15T04:42:29,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/dd56825b3e0f4ba695a745061f5673b3 2024-11-15T04:42:29,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/20783c5a57b741f0a37827c7c8ee6ceb as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/20783c5a57b741f0a37827c7c8ee6ceb 2024-11-15T04:42:29,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/20783c5a57b741f0a37827c7c8ee6ceb, entries=150, sequenceid=315, filesize=30.5 K 2024-11-15T04:42:29,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/e162ed9938024ef1af68b6626fc79739 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/e162ed9938024ef1af68b6626fc79739 2024-11-15T04:42:29,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/e162ed9938024ef1af68b6626fc79739, entries=150, sequenceid=315, filesize=12.0 K 2024-11-15T04:42:29,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/dd56825b3e0f4ba695a745061f5673b3 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/dd56825b3e0f4ba695a745061f5673b3 2024-11-15T04:42:29,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/dd56825b3e0f4ba695a745061f5673b3, entries=150, sequenceid=315, filesize=12.0 K 2024-11-15T04:42:29,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 1f3526eb9bd778c7571f046690073138 in 1693ms, sequenceid=315, compaction requested=true 2024-11-15T04:42:29,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:29,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T04:42:29,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:29,328 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:42:29,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T04:42:29,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:29,328 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:42:29,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1f3526eb9bd778c7571f046690073138:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T04:42:29,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:29,329 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:42:29,329 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/A is initiating minor compaction (all files) 2024-11-15T04:42:29,329 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:42:29,329 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/B is initiating minor compaction (all files) 2024-11-15T04:42:29,329 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/A in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:29,329 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/B in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:29,329 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/eeea2dfe69e34852b81e1de7b82a35a5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/42ab5ed4c17b4bb48cee7798f0a481ff, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/20783c5a57b741f0a37827c7c8ee6ceb] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=92.2 K 2024-11-15T04:42:29,329 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/575401ee82e5430bac6bf5d3261e51a1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/25d6391763d74a8f8e4c51b038ec57a3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/e162ed9938024ef1af68b6626fc79739] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=36.6 K 2024-11-15T04:42:29,329 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:29,329 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. files: [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/eeea2dfe69e34852b81e1de7b82a35a5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/42ab5ed4c17b4bb48cee7798f0a481ff, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/20783c5a57b741f0a37827c7c8ee6ceb] 2024-11-15T04:42:29,330 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 575401ee82e5430bac6bf5d3261e51a1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731645743638 2024-11-15T04:42:29,330 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeea2dfe69e34852b81e1de7b82a35a5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731645743638 2024-11-15T04:42:29,330 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d6391763d74a8f8e4c51b038ec57a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731645744281 2024-11-15T04:42:29,330 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42ab5ed4c17b4bb48cee7798f0a481ff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731645744281 2024-11-15T04:42:29,330 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20783c5a57b741f0a37827c7c8ee6ceb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731645746492 2024-11-15T04:42:29,330 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting e162ed9938024ef1af68b6626fc79739, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731645746492 2024-11-15T04:42:29,336 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:29,337 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#B#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:29,338 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/afc1b28d35fa41418abee49d566df3c7 is 50, key is test_row_0/B:col10/1731645747634/Put/seqid=0 2024-11-15T04:42:29,338 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115379ec469a60245f0bb406b57d5356b8f_1f3526eb9bd778c7571f046690073138 store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:29,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742438_1614 (size=13017) 2024-11-15T04:42:29,344 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115379ec469a60245f0bb406b57d5356b8f_1f3526eb9bd778c7571f046690073138, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:29,344 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115379ec469a60245f0bb406b57d5356b8f_1f3526eb9bd778c7571f046690073138 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:29,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742439_1615 (size=4469) 2024-11-15T04:42:29,748 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/afc1b28d35fa41418abee49d566df3c7 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/afc1b28d35fa41418abee49d566df3c7 2024-11-15T04:42:29,749 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#A#compaction#513 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:29,749 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/ed80309a86bf449bae653a5806e75157 is 175, key is test_row_0/A:col10/1731645747634/Put/seqid=0 2024-11-15T04:42:29,752 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f3526eb9bd778c7571f046690073138/B of 1f3526eb9bd778c7571f046690073138 into afc1b28d35fa41418abee49d566df3c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:29,752 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:29,752 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/B, priority=13, startTime=1731645749328; duration=0sec 2024-11-15T04:42:29,753 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T04:42:29,753 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:B 2024-11-15T04:42:29,753 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T04:42:29,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742440_1616 (size=31971) 2024-11-15T04:42:29,753 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T04:42:29,754 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1541): 1f3526eb9bd778c7571f046690073138/C is initiating minor compaction (all files) 2024-11-15T04:42:29,754 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1f3526eb9bd778c7571f046690073138/C in TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:29,754 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/f81b0a622e124fab94daa6854307a354, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/d4dad1e206294bd595b1ffcc3e343c6d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/dd56825b3e0f4ba695a745061f5673b3] into tmpdir=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp, totalSize=36.6 K 2024-11-15T04:42:29,754 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting f81b0a622e124fab94daa6854307a354, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731645743638 2024-11-15T04:42:29,754 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting d4dad1e206294bd595b1ffcc3e343c6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731645744281 2024-11-15T04:42:29,755 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] compactions.Compactor(225): Compacting dd56825b3e0f4ba695a745061f5673b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731645746492 2024-11-15T04:42:29,757 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/ed80309a86bf449bae653a5806e75157 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ed80309a86bf449bae653a5806e75157 2024-11-15T04:42:29,761 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f3526eb9bd778c7571f046690073138/A of 1f3526eb9bd778c7571f046690073138 into ed80309a86bf449bae653a5806e75157(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:29,761 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:29,761 INFO [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/A, priority=13, startTime=1731645749328; duration=0sec 2024-11-15T04:42:29,761 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:29,761 DEBUG [RS:0;26747e6ada4f:42935-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:A 2024-11-15T04:42:29,763 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1f3526eb9bd778c7571f046690073138#C#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T04:42:29,763 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/ebe012037e67463db24cb7085f218db4 is 50, key is test_row_0/C:col10/1731645747634/Put/seqid=0 2024-11-15T04:42:29,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742441_1617 (size=13017) 2024-11-15T04:42:29,794 INFO [Thread-2362 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2362. 2024-11-15T04:42:29,794 DEBUG [Thread-2362 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:29,794 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:29,794 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:29,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:29,804 INFO [Thread-2366 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2366. 2024-11-15T04:42:29,804 DEBUG [Thread-2366 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:29,804 DEBUG [Thread-2366 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:29,804 DEBUG [Thread-2366 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:29,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:30,179 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/ebe012037e67463db24cb7085f218db4 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/ebe012037e67463db24cb7085f218db4 2024-11-15T04:42:30,187 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1f3526eb9bd778c7571f046690073138/C of 1f3526eb9bd778c7571f046690073138 into ebe012037e67463db24cb7085f218db4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T04:42:30,187 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:30,187 INFO [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., storeName=1f3526eb9bd778c7571f046690073138/C, priority=13, startTime=1731645749328; duration=0sec 2024-11-15T04:42:30,187 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T04:42:30,187 DEBUG [RS:0;26747e6ada4f:42935-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1f3526eb9bd778c7571f046690073138:C 2024-11-15T04:42:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42935 {}] regionserver.HRegion(8855): Flush requested on 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:39,044 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T04:42:39,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T04:42:39,044 INFO [Thread-2364 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2364. 2024-11-15T04:42:39,045 DEBUG [Thread-2364 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:39,045 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:39,045 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:39,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:39,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152bbe46c86cea40aab9b79d6c2393db35_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645747642/Put/seqid=0 2024-11-15T04:42:39,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742442_1618 (size=12454) 2024-11-15T04:42:39,054 INFO [Thread-2360 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2360. 2024-11-15T04:42:39,054 DEBUG [Thread-2360 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:39,054 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:39,055 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:39,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:39,114 INFO [Thread-2358 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2358. 2024-11-15T04:42:39,114 DEBUG [Thread-2358 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T04:42:39,114 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:39,114 DEBUG [Thread-2358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:39,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:39,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T04:42:39,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4006 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3863 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4002 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4014 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3875 2024-11-15T04:42:39,115 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T04:42:39,115 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T04:42:39,115 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T04:42:39,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:39,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:39,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:39,116 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-15T04:42:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T04:42:39,118 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645759118"}]},"ts":"1731645759118"} 2024-11-15T04:42:39,120 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T04:42:39,120 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T04:42:39,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T04:42:39,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, UNASSIGN}] 2024-11-15T04:42:39,122 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, UNASSIGN 2024-11-15T04:42:39,122 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=1f3526eb9bd778c7571f046690073138, regionState=CLOSING, regionLocation=26747e6ada4f,42935,1731645581086 2024-11-15T04:42:39,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, UNASSIGN because future has completed 2024-11-15T04:42:39,123 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T04:42:39,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086}] 2024-11-15T04:42:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T04:42:39,276 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:39,276 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T04:42:39,276 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing 1f3526eb9bd778c7571f046690073138, disabling compactions & flushes 2024-11-15T04:42:39,276 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:39,361 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T04:42:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T04:42:39,453 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:39,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152bbe46c86cea40aab9b79d6c2393db35_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152bbe46c86cea40aab9b79d6c2393db35_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:39,459 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/1d09d7ddce694b86908bddfe6ed165ce, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:39,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/1d09d7ddce694b86908bddfe6ed165ce is 175, key is test_row_0/A:col10/1731645747642/Put/seqid=0 2024-11-15T04:42:39,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742443_1619 (size=31255) 2024-11-15T04:42:39,465 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/1d09d7ddce694b86908bddfe6ed165ce 2024-11-15T04:42:39,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/ea688c0cc6104635a3c121bf18e7cd40 is 50, key is test_row_0/B:col10/1731645747642/Put/seqid=0 2024-11-15T04:42:39,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742444_1620 (size=12301) 2024-11-15T04:42:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T04:42:39,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/ea688c0cc6104635a3c121bf18e7cd40 2024-11-15T04:42:39,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/96463a3df4554ac9aaa9ab7b16b936de is 50, key is test_row_0/C:col10/1731645747642/Put/seqid=0 2024-11-15T04:42:39,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742445_1621 (size=12301) 2024-11-15T04:42:39,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/96463a3df4554ac9aaa9ab7b16b936de 2024-11-15T04:42:39,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/1d09d7ddce694b86908bddfe6ed165ce as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/1d09d7ddce694b86908bddfe6ed165ce 2024-11-15T04:42:39,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/1d09d7ddce694b86908bddfe6ed165ce, entries=150, sequenceid=329, filesize=30.5 K 2024-11-15T04:42:39,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/ea688c0cc6104635a3c121bf18e7cd40 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/ea688c0cc6104635a3c121bf18e7cd40 2024-11-15T04:42:39,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/ea688c0cc6104635a3c121bf18e7cd40, entries=150, sequenceid=329, filesize=12.0 K 2024-11-15T04:42:39,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/96463a3df4554ac9aaa9ab7b16b936de as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/96463a3df4554ac9aaa9ab7b16b936de 2024-11-15T04:42:39,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/96463a3df4554ac9aaa9ab7b16b936de, entries=150, sequenceid=329, filesize=12.0 K 2024-11-15T04:42:39,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=13.42 KB/13740 for 1f3526eb9bd778c7571f046690073138 in 863ms, sequenceid=329, compaction requested=false 2024-11-15T04:42:39,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1f3526eb9bd778c7571f046690073138: 2024-11-15T04:42:39,907 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:39,907 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:39,907 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. after waiting 0 ms 2024-11-15T04:42:39,907 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:39,907 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing 1f3526eb9bd778c7571f046690073138 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-15T04:42:39,911 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111583a88b31d1f24919bc6d09afe7e34f52_1f3526eb9bd778c7571f046690073138 is 50, key is test_row_0/A:col10/1731645759053/Put/seqid=0 2024-11-15T04:42:39,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742446_1622 (size=9914) 2024-11-15T04:42:40,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T04:42:40,315 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T04:42:40,325 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111583a88b31d1f24919bc6d09afe7e34f52_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111583a88b31d1f24919bc6d09afe7e34f52_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:40,326 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/07d79d4218f347a3a309763012fd791e, store: [table=TestAcidGuarantees family=A region=1f3526eb9bd778c7571f046690073138] 2024-11-15T04:42:40,327 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/07d79d4218f347a3a309763012fd791e is 175, key is test_row_0/A:col10/1731645759053/Put/seqid=0 2024-11-15T04:42:40,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742447_1623 (size=22561) 2024-11-15T04:42:40,732 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/07d79d4218f347a3a309763012fd791e 2024-11-15T04:42:40,738 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/6b2df54914b84308a35babf4d575211c is 50, key is test_row_0/B:col10/1731645759053/Put/seqid=0 2024-11-15T04:42:40,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742448_1624 (size=9857) 2024-11-15T04:42:41,143 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/6b2df54914b84308a35babf4d575211c 2024-11-15T04:42:41,156 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/a854b0f4c98247e2a0c5c26a08dc5976 is 50, key is test_row_0/C:col10/1731645759053/Put/seqid=0 2024-11-15T04:42:41,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742449_1625 (size=9857) 2024-11-15T04:42:41,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T04:42:41,562 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/a854b0f4c98247e2a0c5c26a08dc5976 2024-11-15T04:42:41,573 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/A/07d79d4218f347a3a309763012fd791e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/07d79d4218f347a3a309763012fd791e 2024-11-15T04:42:41,579 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/07d79d4218f347a3a309763012fd791e, entries=100, sequenceid=334, filesize=22.0 K 2024-11-15T04:42:41,580 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/B/6b2df54914b84308a35babf4d575211c as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/6b2df54914b84308a35babf4d575211c 2024-11-15T04:42:41,584 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/6b2df54914b84308a35babf4d575211c, entries=100, sequenceid=334, filesize=9.6 K 2024-11-15T04:42:41,584 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/.tmp/C/a854b0f4c98247e2a0c5c26a08dc5976 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/a854b0f4c98247e2a0c5c26a08dc5976 2024-11-15T04:42:41,588 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/a854b0f4c98247e2a0c5c26a08dc5976, entries=100, sequenceid=334, filesize=9.6 K 2024-11-15T04:42:41,589 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 1f3526eb9bd778c7571f046690073138 in 1681ms, sequenceid=334, compaction requested=true 2024-11-15T04:42:41,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b9dee4a50e414b10ba9785736736ddbf, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/a6d7d7435a10401aa3f654fe31f7614d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/c31e79ee672d4147bd923047a8572930, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/23dc851fd0244199bd5d626321c48769, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/10684892722e4140b4a082a7a9679c4a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8b5ba5acbde04edb808adc1cfaf32ec0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b4dbbdba64044af2a9d113dc5249dc03, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/db612b44f5c744e2bef00e4d54059359, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/fd0d822b7da149da913c5d55ec556257, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/9fdb1a380c034fc987d0117c0d097e1f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/06b0d017bea04463b5bf98ef200a2df4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ea5e3bc03bec4211b12a99c0838a4d2d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/560276a83bce4053bc5d0136212ce40c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/4f4467e715cd4a9fbba4cde68b5ccb2c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8a4901e92c114594b8cf56f5fc894c9b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/43774f22cc5844fcaf52212b120a3bb4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/486d595306c14f049612a88e09da7a65, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ed593f06ee7044dd931da7106501645b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/eeea2dfe69e34852b81e1de7b82a35a5, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/42ab5ed4c17b4bb48cee7798f0a481ff, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/20783c5a57b741f0a37827c7c8ee6ceb] to archive 2024-11-15T04:42:41,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:42:41,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b9dee4a50e414b10ba9785736736ddbf to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b9dee4a50e414b10ba9785736736ddbf 2024-11-15T04:42:41,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/a6d7d7435a10401aa3f654fe31f7614d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/a6d7d7435a10401aa3f654fe31f7614d 2024-11-15T04:42:41,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/c31e79ee672d4147bd923047a8572930 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/c31e79ee672d4147bd923047a8572930 2024-11-15T04:42:41,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/23dc851fd0244199bd5d626321c48769 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/23dc851fd0244199bd5d626321c48769 2024-11-15T04:42:41,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/10684892722e4140b4a082a7a9679c4a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/10684892722e4140b4a082a7a9679c4a 2024-11-15T04:42:41,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8b5ba5acbde04edb808adc1cfaf32ec0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8b5ba5acbde04edb808adc1cfaf32ec0 2024-11-15T04:42:41,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b4dbbdba64044af2a9d113dc5249dc03 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/b4dbbdba64044af2a9d113dc5249dc03 2024-11-15T04:42:41,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/db612b44f5c744e2bef00e4d54059359 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/db612b44f5c744e2bef00e4d54059359 2024-11-15T04:42:41,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/fd0d822b7da149da913c5d55ec556257 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/fd0d822b7da149da913c5d55ec556257 2024-11-15T04:42:41,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/9fdb1a380c034fc987d0117c0d097e1f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/9fdb1a380c034fc987d0117c0d097e1f 2024-11-15T04:42:41,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/06b0d017bea04463b5bf98ef200a2df4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/06b0d017bea04463b5bf98ef200a2df4 2024-11-15T04:42:41,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ea5e3bc03bec4211b12a99c0838a4d2d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ea5e3bc03bec4211b12a99c0838a4d2d 2024-11-15T04:42:41,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/560276a83bce4053bc5d0136212ce40c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/560276a83bce4053bc5d0136212ce40c 2024-11-15T04:42:41,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/4f4467e715cd4a9fbba4cde68b5ccb2c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/4f4467e715cd4a9fbba4cde68b5ccb2c 2024-11-15T04:42:41,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8a4901e92c114594b8cf56f5fc894c9b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/8a4901e92c114594b8cf56f5fc894c9b 2024-11-15T04:42:41,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/43774f22cc5844fcaf52212b120a3bb4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/43774f22cc5844fcaf52212b120a3bb4 2024-11-15T04:42:41,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/486d595306c14f049612a88e09da7a65 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/486d595306c14f049612a88e09da7a65 2024-11-15T04:42:41,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ed593f06ee7044dd931da7106501645b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ed593f06ee7044dd931da7106501645b 2024-11-15T04:42:41,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/eeea2dfe69e34852b81e1de7b82a35a5 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/eeea2dfe69e34852b81e1de7b82a35a5 2024-11-15T04:42:41,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/42ab5ed4c17b4bb48cee7798f0a481ff to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/42ab5ed4c17b4bb48cee7798f0a481ff 2024-11-15T04:42:41,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/20783c5a57b741f0a37827c7c8ee6ceb to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/20783c5a57b741f0a37827c7c8ee6ceb 2024-11-15T04:42:41,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7875c96693614b67a2901779659cf320, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77f126ed89c5449f8e1233db88274c4f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/5c85c038f20e4d719356b6cdbf68f4ff, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/3436e5da23e74f6fbc35899244df9a3d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/05908fe6edac49a48b92ce58870fa500, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7a861cac5f4a458f9641993a956179ad, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/4fa310472ee243a6b7521d1a0c8ece0e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77ae11587400456d817fd435b659b7e1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c8d9c4a619c344a2b638192a19a2943f, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/ec064d517cbe406b80383722af05ef5d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/b8802c1088464e63a28265205a54d291, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/dc60881c356a4e998505b6c3eb42632a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c6ff4589eecf403c966eda6fe8d29f3c, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/9d9e0011589b42a4965c2a6bef8d02a0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/35a7e27c9dcf41578a357e81e4a673ea, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/42df6a7308cf42a0916f83de72b1b72d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/d1253d2dbbba4b1a84a3640ae33f321a, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/575401ee82e5430bac6bf5d3261e51a1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/1e41b8782bed40a786394d5c25813d32, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/25d6391763d74a8f8e4c51b038ec57a3, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/e162ed9938024ef1af68b6626fc79739] to archive 2024-11-15T04:42:41,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:42:41,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7875c96693614b67a2901779659cf320 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7875c96693614b67a2901779659cf320 2024-11-15T04:42:41,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77f126ed89c5449f8e1233db88274c4f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77f126ed89c5449f8e1233db88274c4f 2024-11-15T04:42:41,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/5c85c038f20e4d719356b6cdbf68f4ff to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/5c85c038f20e4d719356b6cdbf68f4ff 2024-11-15T04:42:41,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/3436e5da23e74f6fbc35899244df9a3d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/3436e5da23e74f6fbc35899244df9a3d 2024-11-15T04:42:41,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/05908fe6edac49a48b92ce58870fa500 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/05908fe6edac49a48b92ce58870fa500 2024-11-15T04:42:41,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7a861cac5f4a458f9641993a956179ad to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/7a861cac5f4a458f9641993a956179ad 2024-11-15T04:42:41,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/4fa310472ee243a6b7521d1a0c8ece0e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/4fa310472ee243a6b7521d1a0c8ece0e 2024-11-15T04:42:41,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77ae11587400456d817fd435b659b7e1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/77ae11587400456d817fd435b659b7e1 2024-11-15T04:42:41,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c8d9c4a619c344a2b638192a19a2943f to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c8d9c4a619c344a2b638192a19a2943f 2024-11-15T04:42:41,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/ec064d517cbe406b80383722af05ef5d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/ec064d517cbe406b80383722af05ef5d 2024-11-15T04:42:41,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/b8802c1088464e63a28265205a54d291 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/b8802c1088464e63a28265205a54d291 2024-11-15T04:42:41,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/dc60881c356a4e998505b6c3eb42632a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/dc60881c356a4e998505b6c3eb42632a 2024-11-15T04:42:41,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c6ff4589eecf403c966eda6fe8d29f3c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/c6ff4589eecf403c966eda6fe8d29f3c 2024-11-15T04:42:41,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/9d9e0011589b42a4965c2a6bef8d02a0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/9d9e0011589b42a4965c2a6bef8d02a0 2024-11-15T04:42:41,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/35a7e27c9dcf41578a357e81e4a673ea to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/35a7e27c9dcf41578a357e81e4a673ea 2024-11-15T04:42:41,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/42df6a7308cf42a0916f83de72b1b72d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/42df6a7308cf42a0916f83de72b1b72d 2024-11-15T04:42:41,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/d1253d2dbbba4b1a84a3640ae33f321a to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/d1253d2dbbba4b1a84a3640ae33f321a 2024-11-15T04:42:41,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/575401ee82e5430bac6bf5d3261e51a1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/575401ee82e5430bac6bf5d3261e51a1 2024-11-15T04:42:41,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/1e41b8782bed40a786394d5c25813d32 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/1e41b8782bed40a786394d5c25813d32 2024-11-15T04:42:41,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/25d6391763d74a8f8e4c51b038ec57a3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/25d6391763d74a8f8e4c51b038ec57a3 2024-11-15T04:42:41,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/e162ed9938024ef1af68b6626fc79739 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/e162ed9938024ef1af68b6626fc79739 2024-11-15T04:42:41,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/49a3f134279a4967aaacf23cb4c95eea, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/1a5a6dd4b8794a69a69cbc4f6330021d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/539ed8d4b91449baa621359003ed4a89, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/2b9dd1be0ec34f48abc93f9f48b8c8f9, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/057a22b8c39f48c0b78b67066ca67b7e, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/5986e170cba44e5084a455c2ced8ebb4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/b7c869c27c3a4d27bcf7c1765fd96eff, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/87cee007f2074de88dea7c4cb32c009b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/400bafc1f3f547b2a86545f6085dbd02, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/3d001bfe3b0f40849f10ee76644f9810, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/72dff95eaea84333b64d213066f498cb, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e8485818d8744dd098fc886adeddbeb0, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/0425514a4a3843518a9f1354a48651c4, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/b7bf73ed6e77444f84cc81ff7163aa6b, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e27c36c1e414458e8d12aab2b4f91566, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/29c5eab2342a4504856653e6b97f96ff, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/97053b1700b14b559ded82c3b8ec15a7, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/f81b0a622e124fab94daa6854307a354, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/8f8dd196e2f64a238f2f3f8bf0ba3de1, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/d4dad1e206294bd595b1ffcc3e343c6d, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/dd56825b3e0f4ba695a745061f5673b3] to archive 2024-11-15T04:42:41,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T04:42:41,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/49a3f134279a4967aaacf23cb4c95eea to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/49a3f134279a4967aaacf23cb4c95eea 2024-11-15T04:42:41,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/1a5a6dd4b8794a69a69cbc4f6330021d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/1a5a6dd4b8794a69a69cbc4f6330021d 2024-11-15T04:42:41,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/539ed8d4b91449baa621359003ed4a89 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/539ed8d4b91449baa621359003ed4a89 2024-11-15T04:42:41,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/2b9dd1be0ec34f48abc93f9f48b8c8f9 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/2b9dd1be0ec34f48abc93f9f48b8c8f9 2024-11-15T04:42:41,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/057a22b8c39f48c0b78b67066ca67b7e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/057a22b8c39f48c0b78b67066ca67b7e 2024-11-15T04:42:41,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/5986e170cba44e5084a455c2ced8ebb4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/5986e170cba44e5084a455c2ced8ebb4 2024-11-15T04:42:41,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/b7c869c27c3a4d27bcf7c1765fd96eff to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/b7c869c27c3a4d27bcf7c1765fd96eff 2024-11-15T04:42:41,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/87cee007f2074de88dea7c4cb32c009b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/87cee007f2074de88dea7c4cb32c009b 2024-11-15T04:42:41,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/400bafc1f3f547b2a86545f6085dbd02 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/400bafc1f3f547b2a86545f6085dbd02 2024-11-15T04:42:41,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/3d001bfe3b0f40849f10ee76644f9810 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/3d001bfe3b0f40849f10ee76644f9810 2024-11-15T04:42:41,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/72dff95eaea84333b64d213066f498cb to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/72dff95eaea84333b64d213066f498cb 2024-11-15T04:42:41,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e8485818d8744dd098fc886adeddbeb0 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e8485818d8744dd098fc886adeddbeb0 2024-11-15T04:42:41,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/0425514a4a3843518a9f1354a48651c4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/0425514a4a3843518a9f1354a48651c4 2024-11-15T04:42:41,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/b7bf73ed6e77444f84cc81ff7163aa6b to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/b7bf73ed6e77444f84cc81ff7163aa6b 2024-11-15T04:42:41,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e27c36c1e414458e8d12aab2b4f91566 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/e27c36c1e414458e8d12aab2b4f91566 2024-11-15T04:42:41,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/29c5eab2342a4504856653e6b97f96ff to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/29c5eab2342a4504856653e6b97f96ff 2024-11-15T04:42:41,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/97053b1700b14b559ded82c3b8ec15a7 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/97053b1700b14b559ded82c3b8ec15a7 2024-11-15T04:42:41,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/f81b0a622e124fab94daa6854307a354 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/f81b0a622e124fab94daa6854307a354 2024-11-15T04:42:41,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/8f8dd196e2f64a238f2f3f8bf0ba3de1 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/8f8dd196e2f64a238f2f3f8bf0ba3de1 2024-11-15T04:42:41,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/d4dad1e206294bd595b1ffcc3e343c6d to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/d4dad1e206294bd595b1ffcc3e343c6d 2024-11-15T04:42:41,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/dd56825b3e0f4ba695a745061f5673b3 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/dd56825b3e0f4ba695a745061f5673b3 2024-11-15T04:42:41,660 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=4 2024-11-15T04:42:41,661 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. 2024-11-15T04:42:41,661 DEBUG [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for 1f3526eb9bd778c7571f046690073138: Waiting for close lock at 1731645759276Running coprocessor pre-close hooks at 1731645759276Disabling compacts and flushes for region at 1731645759276Disabling writes for close at 1731645759907 (+631 ms)Obtaining lock to block concurrent updates at 1731645759907Preparing flush snapshotting stores in 1f3526eb9bd778c7571f046690073138 at 1731645759907Finished memstore snapshotting TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731645759907Flushing stores of TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138. at 1731645759907Flushing 1f3526eb9bd778c7571f046690073138/A: creating writer at 1731645759907Flushing mob file 1f3526eb9bd778c7571f046690073138/A: appending metadata at 1731645759911 (+4 ms)Flushing mob file 1f3526eb9bd778c7571f046690073138/A: closing flushed file at 1731645759911Flushing 1f3526eb9bd778c7571f046690073138/A: appending metadata at 1731645760326 (+415 ms)Flushing 1f3526eb9bd778c7571f046690073138/A: closing flushed file at 1731645760326Flushing 1f3526eb9bd778c7571f046690073138/B: creating writer at 1731645760735 (+409 ms)Flushing 1f3526eb9bd778c7571f046690073138/B: appending metadata at 1731645760738 (+3 ms)Flushing 1f3526eb9bd778c7571f046690073138/B: closing flushed file at 1731645760738Flushing 1f3526eb9bd778c7571f046690073138/C: creating writer at 1731645761152 (+414 ms)Flushing 1f3526eb9bd778c7571f046690073138/C: appending metadata at 1731645761156 (+4 ms)Flushing 1f3526eb9bd778c7571f046690073138/C: closing flushed file at 1731645761156Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bf7d912: reopening flushed file at 1731645761571 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a951598: reopening flushed file at 1731645761579 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a086843: reopening flushed file at 1731645761584 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 1f3526eb9bd778c7571f046690073138 in 1681ms, sequenceid=334, compaction requested=true at 1731645761589 (+5 ms)Writing region close event to WAL at 1731645761658 (+69 ms)Running coprocessor post-close hooks at 1731645761661 (+3 ms)Closed at 1731645761661 2024-11-15T04:42:41,662 INFO [RS_CLOSE_REGION-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed 1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:41,663 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=1f3526eb9bd778c7571f046690073138, regionState=CLOSED 2024-11-15T04:42:41,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 because future has completed 2024-11-15T04:42:41,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-15T04:42:41,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure 1f3526eb9bd778c7571f046690073138, server=26747e6ada4f,42935,1731645581086 in 2.5420 sec 2024-11-15T04:42:41,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-15T04:42:41,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1f3526eb9bd778c7571f046690073138, UNASSIGN in 2.5450 sec 2024-11-15T04:42:41,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-15T04:42:41,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5480 sec 2024-11-15T04:42:41,671 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731645761670"}]},"ts":"1731645761670"} 2024-11-15T04:42:41,672 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T04:42:41,672 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T04:42:41,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5560 sec 2024-11-15T04:42:43,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T04:42:43,273 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:43,274 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-15T04:42:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:43,278 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:43,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T04:42:43,280 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:43,284 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,288 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C, FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/recovered.edits] 2024-11-15T04:42:43,291 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/07d79d4218f347a3a309763012fd791e to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/07d79d4218f347a3a309763012fd791e 2024-11-15T04:42:43,293 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/1d09d7ddce694b86908bddfe6ed165ce to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/1d09d7ddce694b86908bddfe6ed165ce 2024-11-15T04:42:43,294 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ed80309a86bf449bae653a5806e75157 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/A/ed80309a86bf449bae653a5806e75157 2024-11-15T04:42:43,297 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/6b2df54914b84308a35babf4d575211c to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/6b2df54914b84308a35babf4d575211c 2024-11-15T04:42:43,299 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/afc1b28d35fa41418abee49d566df3c7 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/afc1b28d35fa41418abee49d566df3c7 2024-11-15T04:42:43,300 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/ea688c0cc6104635a3c121bf18e7cd40 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/B/ea688c0cc6104635a3c121bf18e7cd40 2024-11-15T04:42:43,304 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/96463a3df4554ac9aaa9ab7b16b936de to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/96463a3df4554ac9aaa9ab7b16b936de 2024-11-15T04:42:43,305 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/a854b0f4c98247e2a0c5c26a08dc5976 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/a854b0f4c98247e2a0c5c26a08dc5976 2024-11-15T04:42:43,307 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/ebe012037e67463db24cb7085f218db4 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/C/ebe012037e67463db24cb7085f218db4 2024-11-15T04:42:43,312 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/recovered.edits/337.seqid to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138/recovered.edits/337.seqid 2024-11-15T04:42:43,312 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/default/TestAcidGuarantees/1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,313 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T04:42:43,313 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T04:42:43,314 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T04:42:43,318 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115007d47024769419c878eb20d0d730b17_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115007d47024769419c878eb20d0d730b17_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,319 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111517dff034f277429389db37c4300a5240_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111517dff034f277429389db37c4300a5240_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,320 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151b3c343f6e7c41608d21061a5b4f4023_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151b3c343f6e7c41608d21061a5b4f4023_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,321 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115247bbc33aace4486930a6a5623990d2b_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115247bbc33aace4486930a6a5623990d2b_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,322 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152bbe46c86cea40aab9b79d6c2393db35_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152bbe46c86cea40aab9b79d6c2393db35_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,323 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c134572d4974f42b288cc5753d036d1_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152c134572d4974f42b288cc5753d036d1_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,324 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154d4efbc2a97e4ccdabb02ce5eab4aeef_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154d4efbc2a97e4ccdabb02ce5eab4aeef_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,325 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156ac020104dfe4af7bc7d0b1f9dd8fcd1_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156ac020104dfe4af7bc7d0b1f9dd8fcd1_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,326 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157596d087f7574604b57ad5cabaadf6c5_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157596d087f7574604b57ad5cabaadf6c5_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,327 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111583a88b31d1f24919bc6d09afe7e34f52_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111583a88b31d1f24919bc6d09afe7e34f52_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,328 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158e1911f988b04eb2a9f2ea3defaf8fea_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158e1911f988b04eb2a9f2ea3defaf8fea_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,329 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115917e279bcc1e4756853ead987554afbb_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115917e279bcc1e4756853ead987554afbb_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,331 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111593017244cdec449d8dbaa23884621a14_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111593017244cdec449d8dbaa23884621a14_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,332 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115930edf4f81e543c0a5d3398a050c1be1_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115930edf4f81e543c0a5d3398a050c1be1_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,334 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b91c396a516841ca826764ef6921cc3c_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b91c396a516841ca826764ef6921cc3c_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,335 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df1959a597b14ad8911c588c5e6c0968_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115df1959a597b14ad8911c588c5e6c0968_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,336 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ee8c05a0ec754a50a19c447b654cd261_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ee8c05a0ec754a50a19c447b654cd261_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,337 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ef59ae8dac514258b06692eb09c7beee_1f3526eb9bd778c7571f046690073138 to hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ef59ae8dac514258b06692eb09c7beee_1f3526eb9bd778c7571f046690073138 2024-11-15T04:42:43,338 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T04:42:43,341 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:43,344 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T04:42:43,346 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T04:42:43,347 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:43,347 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T04:42:43,348 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731645763347"}]},"ts":"9223372036854775807"} 2024-11-15T04:42:43,350 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T04:42:43,350 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1f3526eb9bd778c7571f046690073138, NAME => 'TestAcidGuarantees,,1731645725122.1f3526eb9bd778c7571f046690073138.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T04:42:43,350 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T04:42:43,350 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731645763350"}]},"ts":"9223372036854775807"} 2024-11-15T04:42:43,352 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T04:42:43,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T04:42:43,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-11-15T04:42:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42655 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T04:42:43,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T04:42:43,388 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T04:42:43,396 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 240), OpenFileDescriptor=441 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=346 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=14368 (was 14631) 2024-11-15T04:42:43,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T04:42:43,396 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T04:42:43,396 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T04:42:43,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:43,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:43,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T04:42:43,397 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T04:42:43,397 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1166144013, stopped=false 2024-11-15T04:42:43,397 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=26747e6ada4f,42655,1731645580243 2024-11-15T04:42:43,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T04:42:43,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T04:42:43,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:42:43,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:42:43,458 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T04:42:43,460 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T04:42:43,460 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T04:42:43,460 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T04:42:43,460 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T04:42:43,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:43,461 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '26747e6ada4f,42935,1731645581086' ***** 2024-11-15T04:42:43,462 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T04:42:43,463 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T04:42:43,463 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T04:42:43,463 INFO [RS:0;26747e6ada4f:42935 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T04:42:43,463 INFO [RS:0;26747e6ada4f:42935 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T04:42:43,463 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(959): stopping server 26747e6ada4f,42935,1731645581086 2024-11-15T04:42:43,463 INFO [RS:0;26747e6ada4f:42935 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T04:42:43,463 INFO [RS:0;26747e6ada4f:42935 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;26747e6ada4f:42935. 2024-11-15T04:42:43,464 DEBUG [RS:0;26747e6ada4f:42935 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T04:42:43,464 DEBUG [RS:0;26747e6ada4f:42935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:43,464 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T04:42:43,464 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T04:42:43,464 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T04:42:43,464 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T04:42:43,465 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T04:42:43,465 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T04:42:43,465 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T04:42:43,465 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T04:42:43,465 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T04:42:43,466 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T04:42:43,466 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T04:42:43,466 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T04:42:43,466 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-15T04:42:43,486 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/info/64e046c8836b4e36a86d35e6d1eb826e is 95, key is TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0./info:/1731645612212/DeleteFamily/seqid=0 2024-11-15T04:42:43,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742450_1626 (size=6514) 2024-11-15T04:42:43,548 INFO [regionserver/26747e6ada4f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T04:42:43,666 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T04:42:43,866 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T04:42:43,891 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/info/64e046c8836b4e36a86d35e6d1eb826e 2024-11-15T04:42:43,917 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/ns/2c35c22aeddd4fadb09f9fb250a4e30d is 93, key is TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0./ns:/1731645612172/DeleteFamily/seqid=0 2024-11-15T04:42:43,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742451_1627 (size=5906) 2024-11-15T04:42:44,066 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T04:42:44,110 INFO [regionserver/26747e6ada4f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T04:42:44,110 INFO [regionserver/26747e6ada4f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T04:42:44,110 INFO [regionserver/26747e6ada4f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-15T04:42:44,266 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T04:42:44,321 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/ns/2c35c22aeddd4fadb09f9fb250a4e30d 2024-11-15T04:42:44,343 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/rep_barrier/2fa8f283e595446595de3658f0212fc0 is 102, key is TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0./rep_barrier:/1731645612172/DeleteFamily/seqid=0 2024-11-15T04:42:44,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742452_1628 (size=6025) 2024-11-15T04:42:44,467 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T04:42:44,667 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T04:42:44,747 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/rep_barrier/2fa8f283e595446595de3658f0212fc0 2024-11-15T04:42:44,776 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/table/e2841d585f5941cca9e2a1b2cc8a933a is 96, key is TestAcidGuarantees,,1731645585722.bffc5fba59ef21ffa1864f468203f7b0./table:/1731645612172/DeleteFamily/seqid=0 2024-11-15T04:42:44,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742453_1629 (size=5936) 2024-11-15T04:42:44,867 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T04:42:45,068 DEBUG [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T04:42:45,180 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/table/e2841d585f5941cca9e2a1b2cc8a933a 2024-11-15T04:42:45,188 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/info/64e046c8836b4e36a86d35e6d1eb826e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/info/64e046c8836b4e36a86d35e6d1eb826e 2024-11-15T04:42:45,193 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/info/64e046c8836b4e36a86d35e6d1eb826e, entries=12, sequenceid=90, filesize=6.4 K 2024-11-15T04:42:45,193 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/ns/2c35c22aeddd4fadb09f9fb250a4e30d as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/ns/2c35c22aeddd4fadb09f9fb250a4e30d 2024-11-15T04:42:45,197 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/ns/2c35c22aeddd4fadb09f9fb250a4e30d, entries=8, sequenceid=90, filesize=5.8 K 2024-11-15T04:42:45,197 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/rep_barrier/2fa8f283e595446595de3658f0212fc0 as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/rep_barrier/2fa8f283e595446595de3658f0212fc0 2024-11-15T04:42:45,201 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/rep_barrier/2fa8f283e595446595de3658f0212fc0, entries=6, sequenceid=90, filesize=5.9 K 2024-11-15T04:42:45,201 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/.tmp/table/e2841d585f5941cca9e2a1b2cc8a933a as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/table/e2841d585f5941cca9e2a1b2cc8a933a 2024-11-15T04:42:45,205 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/table/e2841d585f5941cca9e2a1b2cc8a933a, entries=7, sequenceid=90, filesize=5.8 K 2024-11-15T04:42:45,206 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false 2024-11-15T04:42:45,210 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-15T04:42:45,211 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T04:42:45,211 INFO [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T04:42:45,211 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731645763465Running coprocessor pre-close hooks at 1731645763465Disabling compacts and flushes for region at 1731645763465Disabling writes for close at 1731645763466 (+1 ms)Obtaining lock to block concurrent updates at 1731645763466Preparing flush snapshotting stores in 1588230740 at 1731645763466Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731645763466Flushing stores of hbase:meta,,1.1588230740 at 1731645763467 (+1 ms)Flushing 1588230740/info: creating writer at 1731645763467Flushing 1588230740/info: appending metadata at 1731645763486 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731645763486Flushing 1588230740/ns: creating writer at 1731645763901 (+415 ms)Flushing 1588230740/ns: appending metadata at 1731645763917 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731645763917Flushing 1588230740/rep_barrier: creating writer at 1731645764326 (+409 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731645764343 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731645764343Flushing 1588230740/table: creating writer at 1731645764756 (+413 ms)Flushing 1588230740/table: appending metadata at 1731645764775 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731645764775Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@564b5a27: reopening flushed file at 1731645765187 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42903d26: reopening flushed file at 1731645765193 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fc288c4: reopening flushed file at 1731645765197 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62710483: reopening flushed file at 1731645765201 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false at 1731645765206 (+5 ms)Writing region close event to WAL at 1731645765208 (+2 ms)Running coprocessor post-close hooks at 1731645765211 (+3 ms)Closed at 1731645765211 2024-11-15T04:42:45,211 DEBUG [RS_CLOSE_META-regionserver/26747e6ada4f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T04:42:45,268 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(976): stopping server 26747e6ada4f,42935,1731645581086; all regions closed. 2024-11-15T04:42:45,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741834_1010 (size=25344) 2024-11-15T04:42:45,281 DEBUG [RS:0;26747e6ada4f:42935 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/oldWALs 2024-11-15T04:42:45,281 INFO [RS:0;26747e6ada4f:42935 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26747e6ada4f%2C42935%2C1731645581086.meta:.meta(num 1731645585286) 2024-11-15T04:42:45,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741832_1008 (size=14207030) 2024-11-15T04:42:45,287 DEBUG [RS:0;26747e6ada4f:42935 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/oldWALs 2024-11-15T04:42:45,287 INFO [RS:0;26747e6ada4f:42935 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26747e6ada4f%2C42935%2C1731645581086:(num 1731645584272) 2024-11-15T04:42:45,287 DEBUG [RS:0;26747e6ada4f:42935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T04:42:45,287 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T04:42:45,287 INFO [RS:0;26747e6ada4f:42935 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T04:42:45,287 INFO [RS:0;26747e6ada4f:42935 {}] hbase.ChoreService(370): Chore service for: regionserver/26747e6ada4f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T04:42:45,288 INFO [RS:0;26747e6ada4f:42935 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T04:42:45,288 INFO [regionserver/26747e6ada4f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T04:42:45,288 INFO [RS:0;26747e6ada4f:42935 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42935 2024-11-15T04:42:45,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/26747e6ada4f,42935,1731645581086 2024-11-15T04:42:45,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T04:42:45,381 INFO [RS:0;26747e6ada4f:42935 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T04:42:45,400 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [26747e6ada4f,42935,1731645581086] 2024-11-15T04:42:45,407 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/26747e6ada4f,42935,1731645581086 already deleted, retry=false 2024-11-15T04:42:45,407 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 26747e6ada4f,42935,1731645581086 expired; onlineServers=0 2024-11-15T04:42:45,408 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '26747e6ada4f,42655,1731645580243' ***** 2024-11-15T04:42:45,408 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T04:42:45,408 INFO [M:0;26747e6ada4f:42655 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T04:42:45,408 INFO [M:0;26747e6ada4f:42655 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T04:42:45,408 DEBUG [M:0;26747e6ada4f:42655 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T04:42:45,408 DEBUG [M:0;26747e6ada4f:42655 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T04:42:45,408 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T04:42:45,408 DEBUG [master/26747e6ada4f:0:becomeActiveMaster-HFileCleaner.large.0-1731645583953 {}] cleaner.HFileCleaner(306): Exit Thread[master/26747e6ada4f:0:becomeActiveMaster-HFileCleaner.large.0-1731645583953,5,FailOnTimeoutGroup] 2024-11-15T04:42:45,408 DEBUG [master/26747e6ada4f:0:becomeActiveMaster-HFileCleaner.small.0-1731645583954 {}] cleaner.HFileCleaner(306): Exit Thread[master/26747e6ada4f:0:becomeActiveMaster-HFileCleaner.small.0-1731645583954,5,FailOnTimeoutGroup] 2024-11-15T04:42:45,409 INFO [M:0;26747e6ada4f:42655 {}] hbase.ChoreService(370): Chore service for: master/26747e6ada4f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T04:42:45,409 INFO [M:0;26747e6ada4f:42655 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T04:42:45,409 DEBUG [M:0;26747e6ada4f:42655 {}] master.HMaster(1795): Stopping service threads 2024-11-15T04:42:45,409 INFO [M:0;26747e6ada4f:42655 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T04:42:45,409 INFO [M:0;26747e6ada4f:42655 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T04:42:45,410 ERROR [M:0;26747e6ada4f:42655 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T04:42:45,411 INFO [M:0;26747e6ada4f:42655 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T04:42:45,411 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T04:42:45,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T04:42:45,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T04:42:45,421 DEBUG [M:0;26747e6ada4f:42655 {}] zookeeper.ZKUtil(347): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T04:42:45,422 WARN [M:0;26747e6ada4f:42655 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T04:42:45,422 INFO [M:0;26747e6ada4f:42655 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/.lastflushedseqids 2024-11-15T04:42:45,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742454_1630 (size=72) 2024-11-15T04:42:45,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T04:42:45,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42935-0x1013e1433580001, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T04:42:45,500 INFO [RS:0;26747e6ada4f:42935 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T04:42:45,500 INFO [RS:0;26747e6ada4f:42935 {}] regionserver.HRegionServer(1031): Exiting; stopping=26747e6ada4f,42935,1731645581086; zookeeper connection closed. 2024-11-15T04:42:45,501 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@58f7e956 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@58f7e956 2024-11-15T04:42:45,501 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T04:42:45,831 INFO [M:0;26747e6ada4f:42655 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T04:42:45,831 INFO [M:0;26747e6ada4f:42655 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T04:42:45,831 DEBUG [M:0;26747e6ada4f:42655 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T04:42:45,831 INFO [M:0;26747e6ada4f:42655 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T04:42:45,831 DEBUG [M:0;26747e6ada4f:42655 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T04:42:45,831 DEBUG [M:0;26747e6ada4f:42655 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T04:42:45,831 DEBUG [M:0;26747e6ada4f:42655 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T04:42:45,831 INFO [M:0;26747e6ada4f:42655 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=735.24 KB heapSize=902.29 KB 2024-11-15T04:42:45,850 DEBUG [M:0;26747e6ada4f:42655 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cf1800ebf74438890d5fafe7e86428e is 82, key is hbase:meta,,1/info:regioninfo/1731645585367/Put/seqid=0 2024-11-15T04:42:45,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742455_1631 (size=5672) 2024-11-15T04:42:46,255 INFO [M:0;26747e6ada4f:42655 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2062 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cf1800ebf74438890d5fafe7e86428e 2024-11-15T04:42:46,280 DEBUG [M:0;26747e6ada4f:42655 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b5791967adda4c9e81a77f33334a713b is 2275, key is \x00\x00\x00\x00\x00\x00\x00Z/proc:d/1731645672942/Put/seqid=0 2024-11-15T04:42:46,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742456_1632 (size=42733) 2024-11-15T04:42:46,684 INFO [M:0;26747e6ada4f:42655 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=734.68 KB at sequenceid=2062 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b5791967adda4c9e81a77f33334a713b 2024-11-15T04:42:46,687 INFO [M:0;26747e6ada4f:42655 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b5791967adda4c9e81a77f33334a713b 2024-11-15T04:42:46,700 DEBUG [M:0;26747e6ada4f:42655 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a49492bbae5494da137f9d0cf7f2bcb is 69, key is 26747e6ada4f,42935,1731645581086/rs:state/1731645583999/Put/seqid=0 2024-11-15T04:42:46,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073742457_1633 (size=5156) 2024-11-15T04:42:47,105 INFO [M:0;26747e6ada4f:42655 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2062 (bloomFilter=true), to=hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a49492bbae5494da137f9d0cf7f2bcb 2024-11-15T04:42:47,115 DEBUG [M:0;26747e6ada4f:42655 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4cf1800ebf74438890d5fafe7e86428e as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4cf1800ebf74438890d5fafe7e86428e 2024-11-15T04:42:47,119 INFO [M:0;26747e6ada4f:42655 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4cf1800ebf74438890d5fafe7e86428e, entries=8, sequenceid=2062, filesize=5.5 K 2024-11-15T04:42:47,120 DEBUG [M:0;26747e6ada4f:42655 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b5791967adda4c9e81a77f33334a713b as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b5791967adda4c9e81a77f33334a713b 2024-11-15T04:42:47,123 INFO [M:0;26747e6ada4f:42655 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b5791967adda4c9e81a77f33334a713b 2024-11-15T04:42:47,123 INFO [M:0;26747e6ada4f:42655 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b5791967adda4c9e81a77f33334a713b, entries=162, sequenceid=2062, filesize=41.7 K 2024-11-15T04:42:47,124 DEBUG [M:0;26747e6ada4f:42655 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a49492bbae5494da137f9d0cf7f2bcb as hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5a49492bbae5494da137f9d0cf7f2bcb 2024-11-15T04:42:47,127 INFO [M:0;26747e6ada4f:42655 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38061/user/jenkins/test-data/53ccadd2-fa95-58a6-9eab-cc52d9ce62b6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5a49492bbae5494da137f9d0cf7f2bcb, entries=1, sequenceid=2062, filesize=5.0 K 2024-11-15T04:42:47,128 INFO [M:0;26747e6ada4f:42655 {}] regionserver.HRegion(3140): Finished flush of dataSize ~735.24 KB/752886, heapSize ~901.99 KB/923640, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=2062, compaction requested=false 2024-11-15T04:42:47,130 INFO [M:0;26747e6ada4f:42655 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T04:42:47,130 DEBUG [M:0;26747e6ada4f:42655 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731645765831Disabling compacts and flushes for region at 1731645765831Disabling writes for close at 1731645765831Obtaining lock to block concurrent updates at 1731645765831Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731645765831Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=752886, getHeapSize=923880, getOffHeapSize=0, getCellsCount=2249 at 1731645765832 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731645765832Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731645765832Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731645765850 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731645765850Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731645766260 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731645766280 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731645766280Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731645766688 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731645766700 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731645766700Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18bc58d1: reopening flushed file at 1731645767114 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50953ccc: reopening flushed file at 1731645767120 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a7c45b3: reopening flushed file at 1731645767124 (+4 ms)Finished flush of dataSize ~735.24 KB/752886, heapSize ~901.99 KB/923640, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1297ms, sequenceid=2062, compaction requested=false at 1731645767128 (+4 ms)Writing region close event to WAL at 1731645767130 (+2 ms)Closed at 1731645767130 2024-11-15T04:42:47,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35353 is added to blk_1073741830_1006 (size=888083) 2024-11-15T04:42:47,133 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T04:42:47,133 INFO [M:0;26747e6ada4f:42655 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T04:42:47,133 INFO [M:0;26747e6ada4f:42655 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42655 2024-11-15T04:42:47,133 INFO [M:0;26747e6ada4f:42655 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T04:42:47,179 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T04:42:47,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T04:42:47,282 INFO [M:0;26747e6ada4f:42655 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T04:42:47,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42655-0x1013e1433580000, quorum=127.0.0.1:51527, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T04:42:47,291 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T04:42:47,293 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T04:42:47,294 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T04:42:47,294 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T04:42:47,294 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/hadoop.log.dir/,STOPPED} 2024-11-15T04:42:47,296 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T04:42:47,296 WARN [BP-622654536-172.17.0.2-1731645576553 heartbeating to localhost/127.0.0.1:38061 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T04:42:47,296 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T04:42:47,296 WARN [BP-622654536-172.17.0.2-1731645576553 heartbeating to localhost/127.0.0.1:38061 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-622654536-172.17.0.2-1731645576553 (Datanode Uuid ba099ddd-b7d4-410e-9ed8-42045e285825) service to localhost/127.0.0.1:38061 2024-11-15T04:42:47,300 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/cluster_a5d1cf9b-a665-2596-bed6-8ed0e5e2e7a7/data/data1/current/BP-622654536-172.17.0.2-1731645576553 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T04:42:47,300 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/cluster_a5d1cf9b-a665-2596-bed6-8ed0e5e2e7a7/data/data2/current/BP-622654536-172.17.0.2-1731645576553 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T04:42:47,300 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T04:42:47,307 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T04:42:47,308 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T04:42:47,308 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T04:42:47,308 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T04:42:47,309 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3746fd1c-2d75-5e04-ab55-46c5d69ec14d/hadoop.log.dir/,STOPPED} 2024-11-15T04:42:47,325 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T04:42:47,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down